add focus style to sort select

* break auth panel render into submethods

* move var definition

* break renderUnathorized into submethods

* hide login providers behind dropdown if they are exceed length of 3

* amend dropdown to behave nicely being placed in another dropdown

* add style to providers enclosed in dropdown

* add provider reducer and actions

* add provider save/restore to app flow

* place last login provider first in providers list

* infer StoreState from combineReducers return type

* move collapsed threads retoration to action

* fix: provider lost in other

* add dynamic threshold depending on window width

* fix & add tests

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
+/frontend/node_modules/
-/web/public/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,5 +19,6 @@ debug
 debug.test
 *.prof
 *.test
-/bin/
 remark42
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -5,12 +5,8 @@ workspace:
 pipeline:
   
   build:
-    image: golang:1.10-alpine
+    image: golang:1.12-alpine
     commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
       - cd backend/app
       - go build -v ./...
 
@@ -94,9 +90,3 @@ pipeline:
     secrets: [ email_username, email_password ]
     when:
       status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]
-

.gitignore

@@ -8,9 +8,9 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
+/frontend/node_modules/
-/web/public/
+/frontend/public/
-/web/coverage
+/frontend/coverage
 *.prof
 *.test
 /rest-client.env.json
@@ -18,3 +18,5 @@ debug.test
 .mongo
 remark42
 /bin/
+/backend/var/
+compose-private-backend.yml

.travis.yml

@@ -3,9 +3,6 @@ install:
   - docker-compose --version
 
 script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
   - docker build
       --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
       --build-arg CI=$CI
@@ -19,7 +16,4 @@ script:
       --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
       --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
       --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
       .
-  - docker rm -f mongo
-  

Dockerfile

@@ -19,44 +19,37 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
 
-WORKDIR /go/src/github.com/umputun/remark/backend
+ADD backend /build/backend
-ADD backend /go/src/github.com/umputun/remark/backend
+ADD .git /build/.git
-ADD .git /go/src/github.com/umputun/remark/.git
+WORKDIR /build/backend
+
+ENV GOFLAGS="-mod=vendor"
 
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
-# coverage report
-RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    mkdir -p target && /script/coverage.sh ; \
-    else echo "skip backend coverage" ; fi
-
 # submit coverage to coverals if COVERALLS_TOKEN in env
 RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
     echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+    else goveralls -coverprofile=/profile.cov -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
-    echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
@@ -65,11 +58,12 @@ RUN \
 FROM node:10.11-alpine as build-frontend-deps
 
 ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
+ADD frontend/package.json /srv/frontend/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
@@ -77,9 +71,9 @@ ARG CI
 ARG SKIP_FRONTEND_TEST
 ARG NODE_ENV=production
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD web /srv/web
+ADD frontend /srv/frontend
-RUN cd /srv/web && \
+RUN cd /srv/frontend && \
     if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
     else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
@@ -95,8 +89,8 @@ ADD backend/scripts/restore.sh /usr/local/bin/restore
 ADD backend/scripts/import.sh /usr/local/bin/import
 RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
 RUN ln -s /srv/remark42 /usr/bin/remark42
 

Dockerfile.artifacts

@@ -9,19 +9,20 @@ ARG DRONE_BRANCH
 ENV SKIP_FRONTEND_TEST=true
 
 RUN apk add --no-cache --update git
-ADD web/package.json /srv/web/package.json
+ADD frontend/package.json /srv/frontend/package.json
-ADD web/package-lock.json /srv/web/package-lock.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
-RUN cd /srv/web && CI=true npm ci
+RUN cd /srv/frontend && CI=true npm ci
 
 FROM node:10.11-alpine as build-frontend
 
 ARG CI
 ARG NODE_ENV=production
 ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
 
-COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
-ADD web /srv/web
+ADD frontend /srv/frontend
-RUN cd /srv/web && \
+RUN cd /srv/frontend && \
     npm run build && \
     rm -rf ./node_modules
 
@@ -35,7 +36,7 @@ WORKDIR /go/src/github.com/umputun/remark/backend
 ADD backend /go/src/github.com/umputun/remark/backend
 ADD README.md /go/src/github.com/umputun/remark/
 ADD LICENSE /go/src/github.com/umputun/remark/
-COPY --from=build-frontend /srv/web/public/ web
+COPY --from=build-frontend /srv/frontend/public/ web
 
 RUN \
     export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
@@ -63,32 +64,33 @@ RUN \
     echo "runs outside of drone" && tag=""; \
     else tag=_${DRONE_TAG}; fi && \
     apk add --no-cache --update zip && \
-    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
-    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
-    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
-    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
-    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
 # upload to github
-RUN \
+#RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
+#    if [ -z "$DRONE_TAG" ] ; then \
-    echo "skip upload to github" ; \
+#    echo "skip upload to github" ; \
-    else \
+#    else \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+#    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
 COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/

README.md

@@ -3,23 +3,37 @@
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
 * Social login via Google, Facebook, Github and Yandex
+* Login via email 
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus and wordpress
+* Import from Disqus and WordPress
-* Markdown support
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
+* Telegram notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
 * Self-contained executable can be deployed directly to Linux, Windows and MacOS
-* Clean, lightweight and fully customizable UI
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
+
+<details><summary>Screenshots</summary>
+
+Comments example:
+![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
+
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
+</details>
+
 
 #
 
@@ -29,6 +43,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
       - [Without docker](#without-docker)
       - [Parameters](#parameters)
         - [Required parameters](#required-parameters)
+      - [Quick installation test](#quick-installation-test)
       - [Register oauth2 providers](#register-oauth2-providers)
         - [Google Auth Provider](#google-auth-provider)
         - [GitHub Auth Provider](#github-auth-provider)
@@ -70,7 +85,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 _this is the recommended way to run remark42_
 
 * copy provided `docker-compose.yml` and customize for your needs
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
+* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
 * pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
 * alternatively compile from the sources - `docker-compose build && docker-compose up -d`
 
@@ -83,60 +98,83 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
+| Command line            | Environment             | Default                  | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                           |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
-| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
-| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
-| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                            |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
+| auth.email.host         | AUTH_EMAIL_HOST         |                          | smtp host                                        |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
+| auth.email.port         | AUTH_EMAIL_PORT         | `25`                     | smtp port                                        |
-| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                       |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                    |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                               |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
+| auth.email.tls          | AUTH_EMAIL_TLS          | `false`                  | enable TLS                                       |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
+| auth.email.user         | AUTH_EMAIL_USER         |                          | smtp user name                                   |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
+| auth.email.passwd       | AUTH_EMAIL_PASSWD       |                          | smtp password                                    |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
+| auth.email.timeout      | AUTH_EMAIL_TIMEOUT      | `10s`                    | smtp timeout                                     |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
-| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                             |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                  |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                        |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -170,6 +208,10 @@ services:
             - ./var:/srv/var                        # persistent volume to store all remark42 data
 ```
 
+#### Quick installation test
+
+To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
+
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
@@ -222,6 +264,13 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 
 For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
 
+##### Anonymous Auth Provider
+
+Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
+
+- name should be at least 3 characters long
+- name has to start from the letter and contains letters, numbers, underscores and spaces only.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
@@ -266,7 +315,7 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 
 ```
     environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
+        - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
 To get user id just login and click on your username or any other user you want to promote to admins.
@@ -283,18 +332,28 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['embed'] // optional param; which components to load. default to ["embed"]
+                          // to load all components define components as ['embed', 'last-comments', 'counter']
+                          // available component are:
+                          //     - 'embed': basic comments widget
+                          //     - 'last-comments': last comments widget, see `Last Comments` section below
+                          //     - 'counter': counter widget, see `Counter` section below
     url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -323,19 +382,24 @@ window.REMARK42.changeTheme('light');
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -351,19 +415,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -425,9 +494,9 @@ To run backend with mongodb store mongo container should be started first - `doc
 
 * install [Node.js 8](https://nodejs.org/en/) or higher;
 * install [NPM 6.1.0](https://www.npmjs.com/package/npm);
-* run `npm install` inside `./web`;
+* run `npm install` inside `./frontend`;
 * run `npm run build` there;
-* result files will be saved in `./web/public`.
+* result files will be saved in `./frontend/public`.
 
 **Note** Running `npm install` will set up precommit hooks into your git repository.
 It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
@@ -448,6 +517,10 @@ npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
 Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
 [Redux](https://github.com/zalmoxisus/redux-devtools-extension).
 
+#### Frontend guide
+
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
 ## API
 
 ### Authorization
@@ -481,8 +554,10 @@ type Comment struct {
     User      User            `json:"user"`    // user info, read only
     Locator   Locator         `json:"locator"` // post locator
     Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
+    Vote      int             `json:"vote"`    // vote for the current user, -1/1/0.
+    Controversy float64       `json:"controversy,omitempty"` // comment controversy, read only
     Timestamp time.Time       `json:"time"`    // time stamp, read only
+    Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
     PostTitle string          `json:"title"`   // post title
@@ -492,6 +567,11 @@ type Locator struct {
     SiteID string `json:"site"`     // site id
     URL    string `json:"url"`      // post url
 }
+
+type Edit struct {
+  Timestamp time.Time `json:"time" bson:"time"`
+  Summary   string    `json:"summary"`
+}
 ```
 
 * `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
@@ -518,14 +598,14 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
 ```go
- 	type EditRequest struct {
+   type EditRequest struct {
- 		Text    string `json:"text"`    // updated text
+     Text    string `json:"text"`    // updated text
- 		Summary string `json:"summary"` // optional, summary of the edit
+     Summary string `json:"summary"` // optional, summary of the edit
- 		Delete  bool   `json:"delete"`  // delete flag 
+     Delete  bool   `json:"delete"`  // delete flag
- 	}{}
+   }{}
 ```
 
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
 * `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
 * `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
   ```go
@@ -553,16 +633,59 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go
-  type config struct {
+  type Config struct {
-      Version       string   `json:"version"`
+     	Version        string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
+        EditDuration   int      `json:"edit_duration"`
-      Admins        []string `json:"admins"`
+        MaxCommentSize int      `json:"max_comment_size"`
-      Auth          []string `json:"auth_providers"`
+        Admins         []string `json:"admins"`
-      LowScore      int      `json:"low_score"`
+        AdminEmail     string   `json:"admin_email"`
-      CriticalScore int      `json:"critical_score"`
+        Auth           []string `json:"auth_providers"`
+        LowScore       int      `json:"low_score"`
+        CriticalScore  int      `json:"critical_score"`
+        PositiveScore  bool     `json:"positive_score"`
+        ReadOnlyAge    int      `json:"readonly_age"`
+        MaxImageSize   int      `json:"max_image_size"`
+        EmojiEnabled   bool     `json:"emoji_enabled"`
   }
   ```
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
+
+* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
+
+### Streaming API
+
+Streaming API provide server-sent events for post updates as well as site update
+
+* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
+* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
+
+<details><summary>response example</summary>
+
+```
+data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}         
+```
+
+</details>
 
 ### RSS feeds
 
@@ -570,6 +693,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
@@ -585,7 +715,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
 * `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
 * `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
+* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion.
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
 * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
 * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.

backend/Gopkg.lock

@@ -1,429 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter",
-  ]
-  pruneopts = "UT"
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
-  name = "github.com/globalsign/mgo"
-  packages = [
-    ".",
-    "bson",
-    "internal/json",
-    "internal/sasl",
-    "internal/scram",
-  ]
-  pruneopts = "UT"
-  revision = "113d3961e7311526535a1ef7042196563d442761"
-  version = "r2018.06.15"
-
-[[projects]]
-  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware",
-  ]
-  pruneopts = "UT"
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
-  name = "github.com/go-chi/cors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "dba6525398619dead495962a916728e7ee2ca322"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:d31c72099b9e195785933d11441bc92cce9b2c421f8fca712a7eaf36a8e55a47"
-  name = "github.com/go-pkgz/auth"
-  packages = [
-    ".",
-    "avatar",
-    "logger",
-    "middleware",
-    "provider",
-    "token",
-  ]
-  pruneopts = "UT"
-  revision = "92256b685196d26a346dfbedea12898fe589b0eb"
-  version = "v0.4.0"
-
-[[projects]]
-  digest = "1:1933dabfb0e07548ed9684fdef857cc6f2413bbbb74ba838a7c1f77407b8fd90"
-  name = "github.com/go-pkgz/lgr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8f56fccc1a15dfe9c7f72e049fc87cc378205b97"
-  version = "v0.2.2"
-
-[[projects]]
-  digest = "1:c509e3f646c48148f2239fa66d5cd62785dfd057b5d2b46972e45b60049c19ed"
-  name = "github.com/go-pkgz/mongo"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ee1e9a167094f8cdbc27407907aa694fff350ec3"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
-  name = "github.com/go-pkgz/repeater"
-  packages = [
-    ".",
-    "strategy",
-  ]
-  pruneopts = "UT"
-  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:9aba5c95373481f118e57e9740d9e82f86802d86849809ab89324bcb4f236451"
-  name = "github.com/go-pkgz/rest"
-  packages = [
-    ".",
-    "cache",
-    "logger",
-  ]
-  pruneopts = "UT"
-  revision = "e7d08d0194d613b8854de2e487bf7732500fa153"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = "UT"
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
-  name = "github.com/nullrocks/identicon"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "7875f45b0022edded6377e40639d8aa620193a62"
-
-[[projects]]
-  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  pruneopts = "UT"
-  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
-  version = "v0.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
-  name = "golang.org/x/crypto"
-  packages = [
-    "acme",
-    "acme/autocert",
-  ]
-  pruneopts = "UT"
-  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64",
-  ]
-  pruneopts = "UT"
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom",
-  ]
-  pruneopts = "UT"
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex",
-  ]
-  pruneopts = "UT"
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = "UT"
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  pruneopts = "UT"
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/PuerkitoBio/goquery",
-    "github.com/coreos/bbolt",
-    "github.com/dgrijalva/jwt-go",
-    "github.com/didip/tollbooth",
-    "github.com/didip/tollbooth_chi",
-    "github.com/globalsign/mgo",
-    "github.com/globalsign/mgo/bson",
-    "github.com/go-chi/chi",
-    "github.com/go-chi/chi/middleware",
-    "github.com/go-chi/cors",
-    "github.com/go-chi/render",
-    "github.com/go-pkgz/auth",
-    "github.com/go-pkgz/auth/avatar",
-    "github.com/go-pkgz/auth/provider",
-    "github.com/go-pkgz/auth/token",
-    "github.com/go-pkgz/lgr",
-    "github.com/go-pkgz/mongo",
-    "github.com/go-pkgz/repeater",
-    "github.com/go-pkgz/rest",
-    "github.com/go-pkgz/rest/cache",
-    "github.com/go-pkgz/rest/logger",
-    "github.com/google/uuid",
-    "github.com/gorilla/feeds",
-    "github.com/hashicorp/go-multierror",
-    "github.com/jessevdk/go-flags",
-    "github.com/microcosm-cc/bluemonday",
-    "github.com/patrickmn/go-cache",
-    "github.com/pkg/errors",
-    "github.com/rakyll/statik/fs",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/acme/autocert",
-    "golang.org/x/net/html",
-    "gopkg.in/russross/blackfriday.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/Gopkg.toml

@@ -1,44 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-
-[[constraint]]
-  name = "github.com/globalsign/mgo"
-  version = "r2018.06.15"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-
-
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"

backend/app/cmd/avatar.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"path"
-	"time"
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -31,6 +28,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }
@@ -77,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
 			return nil, err
@@ -92,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
 }
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}

backend/app/cmd/avatar_test.go

@@ -13,32 +13,15 @@ import (
 
 func TestAvatar_Execute(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -46,9 +29,9 @@ func TestAvatar_Execute(t *testing.T) {
 	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
+		"--dst.bolt.file=/tmp/ava-test2.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL)
+	r, err := http.Get(listURL) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL)
+		r, err = http.Get(commentsURL) // nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}
@@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil

backend/app/cmd/cmd.go

@@ -33,7 +33,7 @@ type CommonOpts struct {
 // SetCommon satisfies CommonOptionsCommander interface and sets common option fields
 // The method called by main for each command
 func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
-	c.RemarkURL = commonOpts.RemarkURL
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
 	c.SharedSecret = commonOpts.SharedSecret
 	c.Revision = commonOpts.Revision
 }

backend/app/cmd/server.go

@@ -8,29 +8,33 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	auth_cache "github.com/patrickmn/go-cache"
+	"github.com/kyokomi/emoji"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/provider"
+	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/mongo"
 	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest/api"
 	"github.com/umputun/remark/backend/app/rest/proxy"
+	"github.com/umputun/remark/backend/app/rpc"
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -39,10 +43,11 @@ type ServerCommand struct {
 	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
 	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
 	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Stream StreamGroup `group:"stream" namespace:"stream" env-namespace:"STREAM"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
 	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
@@ -53,23 +58,38 @@ type ServerCommand struct {
 	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
 	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
 	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore   bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
 	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
 	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
 	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
 	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" default:"" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	EnableEmoji     bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
 
 	Auth struct {
 		TTL struct {
 			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
 			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
 		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Google    AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Github    AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Facebook  AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-		Dev      bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Email     struct {
+			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
+			Host         string        `long:"host" env:"HOST" description:"smtp host"`
+			Port         int           `long:"port" env:"PORT" description:"smtp port"`
+			From         string        `long:"from" env:"FROM" description:"email's from"`
+			Subject      string        `long:"subj" env:"SUBJ" default:"remark42 confirmation" description:"email's subject"`
+			ContentType  string        `long:"content-type" env:"CONTENT_TYPE" default:"text/html" description:"content type"`
+			TLS          bool          `long:"tls" env:"TLS" description:"enable TLS"`
+			SMTPUserName string        `long:"user" env:"USER" description:"smtp user name"`
+			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"smtp password"`
+			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"smtp timeout"`
+		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
@@ -83,16 +103,33 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"`
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"TPC"`
+}
+
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
+	FS   struct {
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize      int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+	ResizeWidth  int `long:"resize-width" env:"RESIZE_WIDTH" default:"800" description:"width of resized image"`
+	ResizeHeight int `long:"resize-height" env:"RESIZE_HEIGHT" default:"300" description:"height of resized image"`
 }
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
@@ -104,7 +141,7 @@ type AvatarGroup struct {
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -112,19 +149,14 @@ type CacheGroup struct {
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
-	URL string `long:"url" env:"URL" description:"mongo url"`
-	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"`
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // NotifyGroup defines options for notification
@@ -149,6 +181,21 @@ type SSLGroup struct {
 	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
 }
 
+// StreamGroup define options for streaming apis
+type StreamGroup struct {
+	RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
+	TimeOut         time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
+	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
+}
+
+// RPCGroup defines options for remote modules (plugins)
+type RPCGroup struct {
+	API          string        `long:"api" env:"API" description:"rpc extension api url"`
+	TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+	AuthUser     string        `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -159,6 +206,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -179,6 +227,7 @@ func (s *ServerCommand) Execute(args []string) error {
 	app, err := s.newServerApp()
 	if err != nil {
 		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
 		log.Printf("[ERROR] remark terminated with error %+v", err)
@@ -211,12 +260,20 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
+
 	dataService := &service.DataStore{
-		Interface:              storeEngine,
+		Engine:                 storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
+		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
@@ -250,7 +307,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	}
 
 	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
-	commentFormatter := store.NewCommentFormatter(imgProxy)
+	emojiFmt := store.CommentConverterFunc(func(text string) string { return text })
+	if s.EnableEmoji {
+		emojiFmt = func(text string) string { return emoji.Sprint(text) }
+	}
+	commentFormatter := store.NewCommentFormatter(imgProxy, emojiFmt)
 
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
@@ -272,15 +333,22 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     imageService,
+		Streamer: &api.Streamer{
+			TimeOut:   s.Stream.TimeOut,
+			Refresh:   s.Stream.RefreshInterval,
+			MaxActive: int32(s.Stream.MaxActive),
+		},
+		EmojiEnabled: s.EnableEmoji,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
+		da, errDevAuth := authenticator.DevAuth()
-		if err != nil {
+		if errDevAuth != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -294,6 +362,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -319,12 +388,17 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -363,13 +437,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
-	case "mongo":
+	case "rpc":
-		mgServer, e := s.makeMongo()
+		r := &engine.RPC{Client: rpc.Client{
-		if e != nil {
+			API:        s.Store.RPC.API,
-			return result, errors.Wrap(e, "failed to create mongo server")
+			Client:     http.Client{Timeout: s.Store.RPC.TimeOut},
-		}
+			AuthUser:   s.Store.RPC.AuthUser,
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
+			AuthPasswd: s.Store.RPC.AuthPassword,
-		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
 	}
@@ -385,13 +460,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
-	case "mongo":
-		mgServer, err := s.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
 			return nil, err
@@ -401,6 +469,28 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+				MaxHeight:  s.Image.ResizeHeight,
+				MaxWidth:   s.Image.ResizeWidth,
+			},
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
+		}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
@@ -412,13 +502,14 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 			}
 		}
 		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
-	case "mongo":
+	case "rpc":
-		mgServer, e := s.makeMongo()
+		r := &admin.RPC{Client: rpc.Client{
-		if e != nil {
+			API:        s.Admin.RPC.API,
-			return nil, errors.Wrap(e, "failed to create mongo server")
+			Client:     http.Client{Timeout: s.Admin.RPC.TimeOut},
-		}
+			AuthUser:   s.Admin.RPC.AuthUser,
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
+			AuthPasswd: s.Admin.RPC.AuthPassword,
-		return admin.NewMongoStore(conn, s.SharedSecret), nil
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
 	}
@@ -430,26 +521,33 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
 	case "mem":
 		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
-	// case "mongo":
-	// 	mgServer, err := s.makeMongo()
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "failed to create mongo server")
-	// 	}
-	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
-	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-	// 		cache.MaxKeys(s.Cache.Max.Items))
 	case "none":
 		return &cache.Nop{}, nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
+var msgTemplate = `
-	if s.Mongo.URL == "" {
+<!DOCTYPE html>
-		return nil, errors.New("no mongo URL provided")
+<html>
-	}
+<head>
-	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
+	<meta name="viewport" content="width=device-width" />
-}
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation&nbsp;for <b>{{.User}}</b> on&nbsp;site&nbsp;<b>{{.Site}}</b></p>
+	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
+		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and&nbsp;paste this text into “token” field on&nbsp;comments page</i></p>
+		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+	</div>
+	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
+</div>
+</body>
+</html>
+`
 
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
@@ -471,10 +569,45 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 	if s.Auth.Dev {
+		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
 
+	if s.Auth.Email.Enable {
+		params := sender.EmailParams{
+			Host:         s.Auth.Email.Host,
+			Port:         s.Auth.Email.Port,
+			From:         s.Auth.Email.From,
+			Subject:      s.Auth.Email.Subject,
+			ContentType:  s.Auth.Email.ContentType,
+			TLS:          s.Auth.Email.TLS,
+			SMTPUserName: s.Auth.Email.SMTPUserName,
+			SMTPPassword: s.Auth.Email.SMTPPassword,
+			TimeOut:      s.Auth.Email.TimeOut,
+		}
+		sndr := sender.NewEmailClient(params, log.Default())
+		authenticator.AddVerifProvider("email", msgTemplate, sndr)
+	}
+
+	if s.Auth.Anonymous {
+		log.Print("[INFO] anonymous access enabled")
+		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
+		authenticator.AddDirectProvider("anonymous", provider.CredCheckerFunc(func(user, _ string) (ok bool, err error) {
+			user = strings.TrimSpace(user)
+			if len(user) < 3 {
+				log.Printf("[WARN] name %q is too short, should be at least 3 characters", user)
+				return false, nil
+			}
+
+			if !isValidAnonName(user) {
+				log.Printf("[WARN] name %q should have letters, digits, underscores and spaces only", user)
+				return false, nil
+			}
+			return true, nil
+		}))
+	}
+
 	if providers == 0 {
 		log.Printf("[WARN] no auth providers defined")
 	}
@@ -551,11 +684,13 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
 		Logger:            log.Default(),
 		RefreshCache:      newAuthRefreshCache(),
+		UseGravatar:       true,
 	})
 	s.addAuthProviders(authenticator)
 	return authenticator
@@ -563,17 +698,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*auth_cache.Cache
+	*authcache.Cache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
 }
 
+// Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 	return c.Cache.Get(key.(string))
 }
 
+// Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
 }

backend/app/cmd/server_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
@@ -12,28 +13,27 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
-	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
+	"github.com/jessevdk/go-flags"
-	flags "github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	port := rand.Intn(40000) + 10000
-		o.Port = 18080
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	time.Sleep(250 * time.Millisecond) // let server start
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -43,24 +43,27 @@ func TestServerApp(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 	body, _ = ioutil.ReadAll(resp.Body)
 	t.Log(string(body))
 
-	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+	email, err := app.dataService.AdminStore.Email("")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
@@ -72,7 +75,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -83,52 +86,22 @@ func TestServerApp_DevMode(t *testing.T) {
 	app.Wait()
 }
 
-func TestServerApp_WithMongo(t *testing.T) {
+func TestServerApp_AnonMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = port
+		o.Auth.Anonymous = true
+		return o
+	})
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
-
-	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
-	// prepare options
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
-		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
-	require.Nil(t, err)
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation = "/tmp"
-
-	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
-
-	defer func() {
-		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, err)
-		conn := mongo.NewConnection(s, "test_remark", "")
-		_ = conn.WithDB(func(dbase *mgo.Database) error {
-			assert.NoError(t, dbase.DropDatabase())
-			return nil
-		})
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
 	go func() { _ = app.run(ctx) }()
 	time.Sleep(100 * time.Millisecond) // let server start
 
+	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider for anon")
+	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
+
 	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -136,6 +109,24 @@ func TestServerApp_WithMongo(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "pong", string(body))
 
+	// try to login with good name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+
+	// try to login with bad name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
+	// try to login with short name
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 403, resp.StatusCode)
+
 	app.Wait()
 }
 
@@ -145,8 +136,10 @@ func TestServerApp_WithSSL(t *testing.T) {
 
 	// prepare options
 	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
+		"--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem",
+		"--ssl.port=18443", "--image.fs.path=/tmp"})
 	require.Nil(t, err)
 
 	// create app
@@ -193,6 +186,45 @@ func TestServerApp_WithSSL(t *testing.T) {
 	app.Wait()
 }
 
+func TestServerApp_WithRemote(t *testing.T) {
+
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
+		"--port=12345", "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+	require.Nil(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+	// create app
+	app, err := opts.newServerApp()
+	require.Nil(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(5 * time.Second)
+		log.Print("[TEST] terminate app")
+		cancel()
+	}()
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	// send ping
+	resp, err := http.Get("http://localhost:12345/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	app.Wait()
+}
+
 func TestServerApp_Failed(t *testing.T) {
 	opts := ServerCommand{}
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
@@ -200,7 +232,7 @@ func TestServerApp_Failed(t *testing.T) {
 	p := flags.NewParser(&opts, flags.Default)
 
 	// RO bolt location
-	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null", "--image.fs.path=/tmp"})
 	assert.Nil(t, err)
 	_, err = opts.newServerApp()
 	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
@@ -240,7 +272,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 func TestServerApp_Shutdown(t *testing.T) {
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+		o.Port = rand.Intn(40000) + 10000
 		return o
 	})
 	st := time.Now()
@@ -264,7 +296,7 @@ func TestServerApp_MainSignal(t *testing.T) {
 
 	p := flags.NewParser(&s, flags.Default)
 	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
-		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none", "--image.fs.path=/tmp"}
 	defer os.Remove("/tmp/ava-test.db")
 	_, err := p.ParseArgs(args)
 	require.Nil(t, err)
@@ -316,8 +348,9 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
-	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
+	port := rand.Intn(40000) + 10000
-		o.Port = 18080
+	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
+		o.Port = port
 		return o
 	})
 
@@ -345,18 +378,19 @@ func TestServerAuthHooks(t *testing.T) {
 	t.Log(tk)
 
 	// add comment
-	client := http.Client{Timeout: 5 * time.Second}
+	client := http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
+	req, e := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
 	assert.Nil(t, e)
 	req.SetBasicAuth("admin", "password")
 	resp, e = client.Do(req)
@@ -370,10 +404,10 @@ func TestServerAuthHooks(t *testing.T) {
 	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
 
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	defer resp.Body.Close()
@@ -390,7 +424,7 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	p := flags.NewParser(&cmd, flags.Default)
 	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
 	require.Nil(t, err)
-	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation, cmd.Image.FS.Path = "/tmp", "fs", "/tmp", "/tmp"
 	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
 	cmd.Store.Bolt.Timeout = 10 * time.Second
 	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
@@ -410,10 +444,10 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	require.Nil(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
+	time.AfterFunc(duration, func() {
-		time.Sleep(duration)
 		log.Print("[TEST] terminate app")
 		cancel()
-	}()
+	})
+	rand.Seed(time.Now().UnixNano())
 	return app, ctx
 }

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )
@@ -62,7 +62,7 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 		return
 	}
 	log.Setup(log.Msec, log.LevelBraces)

backend/app/main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -11,20 +12,25 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(500 * time.Millisecond)
+		time.Sleep(5000 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		require.Nil(t, e)
 	}()
 
 	wg := sync.WaitGroup{}
@@ -32,20 +38,29 @@ func TestMain(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		assert.True(t, time.Since(st).Seconds() >= 5, "should take about 5s")
 		wg.Done()
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	var passed bool
+	err = repeater.NewDefault(10, time.Millisecond*500).Do(context.Background(), func() error {
+		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		if e != nil {
+			t.Logf("%+v", e)
+			return e
+		}
+		require.Nil(t, e)
+		defer resp.Body.Close()
+		assert.Equal(t, 200, resp.StatusCode)
+		body, e := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, e)
+		assert.Equal(t, "pong", string(body))
+		passed = true
+		return nil
+	})
 
-	// send ping
+	assert.NoError(t, err)
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
+	assert.Equal(t, true, passed, "at least on ping passed")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
 
 	wg.Wait()
 }

backend/app/migrator/backup_test.go

@@ -16,7 +16,8 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {

backend/app/migrator/disqus.go

@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue

backend/app/migrator/disqus_test.go

@@ -20,13 +20,13 @@ func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	d := Disqus{DataStore: &dataStore}
 	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
 	assert.Nil(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
 	assert.Nil(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 

backend/app/migrator/migrator.go

@@ -27,7 +27,7 @@ type Exporter interface {
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
@@ -42,6 +42,8 @@ type ImportParams struct {
 	SiteID    string
 }
 
+var adminUser = store.User{Admin: true}
+
 // ImportComments imports from given provider format and saves to store
 func ImportComments(p ImportParams) (int, error) {
 	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)

backend/app/migrator/migrator_test.go

@@ -4,11 +4,13 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
@@ -25,7 +27,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
@@ -35,7 +37,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
@@ -51,7 +53,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
@@ -61,7 +63,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
@@ -80,7 +82,7 @@ func TestMigrator_ImportNative(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -91,7 +93,7 @@ func TestMigrator_ImportNative(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 2, size)
 
-	last, err := dataStore.Last("radio-t", 10)
+	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
@@ -100,7 +102,7 @@ func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	dataStore := &service.DataStore{Engine: b}
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -2,23 +2,28 @@ package migrator
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
+	"sync/atomic"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/syncs"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
+const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
 // {"version": 1, comments:[{...}\n,{}], meta: {meta}}
 // each comments starts from the new line
 type Native struct {
-	DataStore Store
+	DataStore  Store
+	Concurrent int
 }
 
 type meta struct {
@@ -44,8 +49,8 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	commentsCount := 0
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
-		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -70,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -91,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -99,8 +104,13 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, err
 	}
 
-	failed := 0
+	var failed, total, comments int64
-	total, comments := 0, 0
+
+	concurrent := defaultConcurrent
+	if n.Concurrent > 0 {
+		concurrent = n.Concurrent
+	}
+	grp := syncs.NewSizedGroup(concurrent, syncs.Preemptive)
 
 	for {
 		comment := store.Comment{}
@@ -112,27 +122,34 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		total++
 
 		if err != nil {
+			atomic.AddInt64(&failed, 1)
 			failed++
 			continue
 		}
 
-		if _, err = n.DataStore.Create(comment); err != nil {
+		// write comments in parallel
-			failed++
+		grp.Go(func(context.Context) {
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
+			if _, e := n.DataStore.Create(comment); e != nil {
-			continue
+				atomic.AddInt64(&failed, 1)
-		}
+				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
-		comments++
+				return
-		if comments%1000 == 0 {
+			}
-			log.Printf("[DEBUG] imported %d comments", comments)
+			num := atomic.AddInt64(&comments, 1)
-		}
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
+			}
+		})
+
 	}
 
+	grp.Wait()
+
 	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
+		return int(comments), errors.Errorf("failed to save %d comments", failed)
 	}
 	log.Printf("[INFO] imported %d comments from %d records", comments, total)
 
 	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
 
-	return comments, err
+	return int(comments), err
 }

backend/app/migrator/native_test.go

@@ -40,25 +40,25 @@ func TestNative_Export(t *testing.T) {
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
-	meta := struct {
+	m := struct {
 		Version int                    `json:"version"`
 		Users   []service.UserMetaData `json:"users"`
 		Posts   []service.PostMetaData `json:"posts"`
 	}{}
 
-	require.NoError(t, dec.Decode(&meta), "decode meta")
+	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(meta.Users))
+	assert.Equal(t, 2, len(m.Users))
-	assert.Equal(t, "user1", meta.Users[0].ID)
+	assert.Equal(t, "user1", m.Users[0].ID)
-	assert.Equal(t, false, meta.Users[0].Blocked.Status)
+	assert.Equal(t, false, m.Users[0].Blocked.Status)
-	assert.Equal(t, true, meta.Users[0].Verified)
+	assert.Equal(t, true, m.Users[0].Verified)
-	assert.Equal(t, "user2", meta.Users[1].ID)
+	assert.Equal(t, "user2", m.Users[1].ID)
-	assert.Equal(t, true, meta.Users[1].Blocked.Status)
+	assert.Equal(t, true, m.Users[1].Blocked.Status)
-	assert.Equal(t, false, meta.Users[1].Verified)
+	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(meta.Posts))
+	assert.Equal(t, 1, len(m.Posts))
-	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
+	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
-	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
 	comments := [3]store.Comment{}
 
@@ -77,12 +77,13 @@ func TestNative_Import(t *testing.T) {
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, size)
 
-	comments, err := b.Last("radio-t", 10)
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
@@ -106,7 +107,8 @@ func TestNative_ImportWrongVersion(t *testing.T) {
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
 	assert.Equal(t, 0, size)
@@ -126,11 +128,12 @@ func TestNative_ImportManyWithError(t *testing.T) {
 	buf.WriteString("{}\n")
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
 	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1200, len(comments))
 }
@@ -142,7 +145,7 @@ func prep(t *testing.T) *service.DataStore {
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
 	assert.Nil(t, err)
 
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",

backend/app/migrator/wordpress.go

@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue
@@ -142,7 +143,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -17,17 +17,17 @@ import (
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
 	assert.Nil(t, err, "create store")
 
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.Nil(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last(siteID, 10)
+	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 

backend/app/notify/notify.go

@@ -29,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
-	Get(locator store.Locator, id string) (store.Comment, error)
+	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment
@@ -68,7 +69,7 @@ func (s *Service) Submit(comment store.Comment) {
 	}
 	parentComment := store.Comment{}
 	if s.dataService != nil {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
+		if p, err := s.dataService.Get(comment.Locator, comment.ParentID, store.User{}); err == nil {
 			parentComment = p
 		}
 	}

backend/app/notify/notify_test.go

@@ -146,7 +146,7 @@ func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v",
 
 type mockStore struct{ data map[string]store.Comment }
 
-func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+func (m *mockStore) Get(_ store.Locator, id string, user store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
 		return store.Comment{}, errors.New("no such id")

backend/app/notify/telegram.go

@@ -42,7 +42,10 @@ func NewTelegram(token string, channelID string, timeout time.Duration, api stri
 	}
 	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
 
-	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	err := repeater.NewDefault(5, time.Millisecond*250).Do(ctx, func() error {
 		client := http.Client{Timeout: telegramTimeOut}
 		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
 		if err != nil {
@@ -110,10 +113,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/notify/telegram_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NotNil(t, err, "should failed")
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())

backend/app/rest/api/admin.go

@@ -15,35 +15,29 @@ import (
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService   *service.DataStore
+	dataService   adminStore
 	cache         cache.LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
+type adminStore interface {
-	router := chi.NewRouter()
+	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
-	router.Use(middlewares...)
+	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
-	router.Put("/user/{userid}", a.setBlockCtrl)
+	IsBlocked(siteID string, userID string) bool
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
+	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
+	Blocked(siteID string) ([]store.BlockedUser, error)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
+	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
-	router.Put("/pin/{id}", a.setPinCtrl)
+	SetVerified(siteID string, userID string, status bool) error
-	router.Get("/blocked", a.blockedUsersCtrl)
+	SetReadOnly(locator store.Locator, status bool) error
-	router.Put("/readonly", a.setReadOnlyCtrl)
+	SetPin(locator store.Locator, commentID string, status bool) error
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
 }
 
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
@@ -55,10 +49,10 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	err := a.dataService.Delete(locator, id, store.SoftDelete)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment", rest.ErrInternal)
 		return
 	}
-	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
@@ -70,8 +64,8 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+	if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
@@ -86,9 +80,9 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
 
-	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0, rest.GetUserOrEmpty(r))
 	if err != nil || len(ucomments) == 0 {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
 	}
 	render.Status(r, http.StatusOK)
@@ -103,7 +97,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token", rest.ErrActionRejected)
 		return
 	}
 
@@ -111,19 +105,19 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
 	if !claims.User.BoolAttr("delete_me") {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token", rest.ErrNoAccess)
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
+		avatarStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
 	}
@@ -147,9 +141,16 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
+
+	// delete comments for permanently blocked user.
+	if blockStatus && ttl == time.Duration(0) {
+		if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
+			log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
+		}
+	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
@@ -159,7 +160,7 @@ func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	users, err := a.dataService.Blocked(siteID)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users", rest.ErrSiteNotFound)
 		return
 	}
 	render.JSON(w, r, users)
@@ -178,13 +179,14 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
 	if !roStatus {
 		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
+			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+				"read-only due the age", rest.ErrActionRejected)
 			return
 		}
 	}
 
 	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status", rest.ErrPostNotFound)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
@@ -198,7 +200,7 @@ func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
 
 	c, err := a.dataService.SetTitle(locator, id)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title", rest.ErrInternal)
 		return
 	}
 	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
@@ -215,7 +217,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 	verifyStatus := r.URL.Query().Get("verified") == "1"
 
 	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
@@ -230,48 +232,9 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	pinStatus := r.URL.Query().Get("pin") == "1"
 
 	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status", rest.ErrActionRejected)
 		return
 	}
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

backend/app/rest/api/admin_test.go

@@ -16,12 +16,13 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/auth/token"
 	R "github.com/go-pkgz/rest"
-	"github.com/umputun/remark/backend/app/store/service"
+	"github.com/go-pkgz/rest/cache"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestAdmin_Delete(t *testing.T) {
@@ -36,12 +37,32 @@ func TestAdmin_Delete(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := http.Client{}
+	// check last comments
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments := []store.Comment{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+
+	// check multi count
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -52,6 +73,36 @@ func TestAdmin_Delete(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
+
+	time.Sleep(250 * time.Millisecond)
+	// check last comments updated
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+
+	// check count updated
+	res, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah")
+	assert.Equal(t, 200, code)
+	b := map[string]interface{}{}
+	err = json.Unmarshal([]byte(res), &b)
+	assert.Nil(t, err)
+	t.Logf("%#v", b)
+	assert.Equal(t, 1.0, b["count"], "should report 1 comments")
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestAdmin_Title(t *testing.T) {
@@ -61,11 +112,13 @@ func TestAdmin_Title(t *testing.T) {
 	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -80,13 +133,12 @@ func TestAdmin_Title(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
@@ -116,38 +168,37 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	_, err = srv.DataService.Create(c3)
 	assert.NoError(t, err)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
+	cmntWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
+	err = json.Unmarshal([]byte(res), &cmntWithInfo)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
+	assert.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
 
 	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
+	assert.Equal(t, id1, cmntWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
+	assert.Equal(t, "o test test #1", cmntWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
+	assert.False(t, cmntWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
+	t.Logf("%+v", cmntWithInfo.Comments[0].User)
 
 	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
+	assert.True(t, cmntWithInfo.Comments[1].Deleted)
 
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
+	assert.True(t, cmntWithInfo.Comments[2].Deleted)
 }
 
 func TestAdmin_Pin(t *testing.T) {
@@ -167,6 +218,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
+		requireAdminOnly(t, req)
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
@@ -197,26 +249,27 @@ func TestAdmin_Block(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	makeTwoComments := func() {
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+		c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+		c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
-	_, err := srv.DataService.Create(c1)
+		_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+		require.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
+		_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+		require.Nil(t, err)
+	}
 
 	block := func(val int, ttl string) (code int, body []byte) {
-		client := http.Client{}
 		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
 		if ttl != "" {
 			url = url + "&ttl=" + ttl
 		}
 		req, e := http.NewRequest(http.MethodPut, url, nil)
 		assert.Nil(t, e)
-		req.SetBasicAuth("admin", "password")
+		requireAdminOnly(t, req)
-		resp, e := client.Do(req)
+		resp, e := sendReq(t, req, adminUmputunToken)
 		require.Nil(t, e)
 		body, e = ioutil.ReadAll(resp.Body)
 		assert.Nil(t, e)
@@ -224,25 +277,49 @@ func TestAdmin_Block(t *testing.T) {
 		return resp.StatusCode, body
 	}
 
+	makeTwoComments()
+
 	// block permanently
 	code, body := block(1, "")
 	require.Equal(t, 200, code)
 	j := R.JSON{}
-	err = json.Unmarshal(body, &j)
+	err := json.Unmarshal(body, &j)
 	assert.Nil(t, err)
 	assert.Equal(t, "user1", j["user_id"])
 	assert.Equal(t, true, j["block"])
 	assert.Equal(t, "radio-t", j["site_id"])
 
+	assert.True(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
+
+	// get last to confirm one comment deleted
+	bodyStr, code := get(t, ts.URL+"/api/v1/last/10?site=radio-t")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	assert.NoError(t, json.Unmarshal([]byte(bodyStr), &pi))
+	assert.Equal(t, 1, len(pi), "last status updated, one comment left")
+
+	// check if count call has one comment left
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	pi = []store.PostInfo{}
+	err = json.Unmarshal(body, &pi)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}), pi)
+
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[0].Text, "permanent block clear comment")
-	assert.True(t, comments.Comments[0].Deleted)
+	assert.True(t, comments.Comments[0].Deleted, "permanent block set deleted comment's status")
 
+	// unblock
 	code, body = block(-1, "")
 	require.Equal(t, 200, code)
 	err = json.Unmarshal(body, &j)
@@ -250,77 +327,91 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, false, j["block"])
 
 	// block with ttl
+	makeTwoComments()
 	code, _ = block(1, "50ms")
 	require.Equal(t, 200, code)
 
+	// get as regular user
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
-	assert.True(t, comments.Comments[0].Deleted)
+	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
+	srv.pubRest.cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
+	assert.Equal(t, "test test #1", comments.Comments[2].Text, "restored")
-	assert.False(t, comments.Comments[0].Deleted)
+	assert.False(t, comments.Comments[2].Deleted)
+
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
 }
 
 func TestAdmin_BlockedList(t *testing.T) {
-	ts, _, teardown := startupT(t)
+	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2 name", ID: "user2"}}
+
+	// write comments for user1 and user2
+	_, err := srv.DataService.Create(c1)
+	assert.Nil(t, err)
+	_, err = srv.DataService.Create(c2)
+	assert.Nil(t, err)
 
 	// block user1
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err := sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	res, err := client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(users), "two users blocked")
 	assert.Equal(t, "user1", users[0].ID)
+	assert.Equal(t, "user1 name", users[0].Name)
 	assert.Equal(t, "user2", users[1].ID)
-
+	assert.Equal(t, "user2 name", users[1].Name)
+	t.Logf("%+v", users)
 	time.Sleep(50 * time.Millisecond)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	res, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(users), "one user left blocked")
-
 }
 
 func TestAdmin_ReadOnly(t *testing.T) {
@@ -341,15 +432,15 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err := sendReq(t, req, "") // non-admin user
-	resp, err := client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
@@ -361,20 +452,18 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err := json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
-	assert.Nil(t, err)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
@@ -385,13 +474,37 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err = json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
-	assert.Nil(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
+func TestAdmin_ReadOnlyNoComments(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	// set post to read-only
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+	assert.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	assert.NotNil(t, err)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&format=tree")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
+}
+
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -406,30 +519,27 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, info.ReadOnly, "ro by age")
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 }
@@ -450,13 +560,13 @@ func TestAdmin_Verify(t *testing.T) {
 	verified := srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	_, err = client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.True(t, verified)
 
@@ -472,9 +582,9 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
@@ -519,12 +629,11 @@ func TestAdmin_ExportFile(t *testing.T) {
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	assert.Equal(t, 200, resp.StatusCode)
 	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
@@ -552,7 +661,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	assert.Nil(t, err)
 
-	comments, err := srv.DataService.User("radio-t", "user1", 0, 0)
+	comments, err := srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments), "a comment for user1")
 
@@ -583,12 +692,13 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
+
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	_, err = srv.DataService.User("radio-t", "user1", 0, 0)
+	_, err = srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
 	assert.EqualError(t, err, "no comments for user user1 in store")
 }
 

backend/app/rest/api/migrator.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -40,15 +39,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
@@ -56,13 +46,14 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	tmpfile, err := m.saveTemp(r.Body)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -78,25 +69,26 @@ func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 
 	if m.isBusy(siteID) {
-		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"),
+			"import rejected", rest.ErrActionRejected)
 		return
 	}
 
 	if err := r.ParseMultipartForm(20 * 1024 * 1024); err != nil { // 20M max memory, if bigger will make a file
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
 		return
 	}
 
 	file, _, err := r.FormFile("file")
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import from the request")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import file from the request", rest.ErrInternal)
 		return
 	}
 	defer func() { _ = file.Close() }()
 
 	tmpfile, err := m.saveTemp(file)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file", rest.ErrInternal)
 		return
 	}
 
@@ -155,7 +147,7 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed", rest.ErrInternal)
 		return
 	}
 }

backend/app/rest/api/migrator_test.go

@@ -3,43 +3,30 @@ package api
 import (
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/migrator"
-	"github.com/umputun/remark/backend/app/store"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -50,19 +37,12 @@ func TestMigrator_Import(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
@@ -77,7 +57,7 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=radio-t&provider=native", contentType, bodyBuf)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
@@ -85,23 +65,17 @@ func TestMigrator_ImportForm(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client := &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 }
+
 func TestMigrator_ImportFromWP(t *testing.T) {
-	srv, ds, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress", r)
 	assert.Nil(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
@@ -113,40 +87,18 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
 
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.NoError(t, ds.Interface.Close())
-
-	tsAccess, _, teardownAccess := startupT(t)
-	defer teardownAccess()
-
-	res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
-	require.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-	t.Logf("%+v", comments)
-	assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
-		comments[0].Text)
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
@@ -154,18 +106,18 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -173,28 +125,29 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusConflict, resp.StatusCode)
-
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
 	recs := []string{}
-	for i := 0; i < 5000; i++ {
+	for i := 0; i < 150; i++ {
 		recs = append(recs, fmt.Sprintf(tmpl, i))
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
 	resp, err := client.Do(req)
@@ -202,40 +155,36 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t&timeout=100ms", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
 	assert.NoError(t, err)
 	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+
+	waitForImportCompletion(t, ts)
 }
 
 func TestMigrator_Export(t *testing.T) {
-	srv, _, ts := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanupImportSrv(srv, ts)
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusAccepted, resp.StatusCode)
-	client = &http.Client{Timeout: 10 * time.Second}
+	waitForImportCompletion(t, ts)
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
-	req.SetBasicAuth("admin", "password")
-	assert.NoError(t, err)
-	resp, err = client.Do(req)
-	assert.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -252,7 +201,7 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -266,42 +215,26 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t", nil)
 	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
 }
 
-func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
+func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
+	client := &http.Client{Timeout: 10 * time.Second}
-	require.Nil(t, err)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t", nil)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	require.NoError(t, err)
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
+	req.SetBasicAuth("admin", "password")
-	svc = &Migrator{
+	assert.NoError(t, err)
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
+	resp, err := client.Do(req)
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
+	require.NoError(t, err)
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
+	assert.Equal(t, 200, resp.StatusCode)
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
+	b, err := ioutil.ReadAll(resp.Body)
-		Cache:             &cache.Nop{},
+	require.NoError(t, err)
-		KeyStore:          adminStore,
+	defer resp.Body.Close()
-	}
+	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	return svc, dataStore, ts
-}
-
-func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
-	ts.Close()
-	_ = os.Remove(testDb)
 }
 
 var xmlTestWP = `

backend/app/rest/api/rest.go

@@ -5,9 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
 	"strings"
 	"sync"
 	"time"
@@ -30,6 +28,7 @@ import (
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -44,6 +43,8 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     *image.Service
+	Streamer         *Streamer
 
 	WebRoot         string
 	RemarkURL       string
@@ -54,13 +55,17 @@ type Rest struct {
 		Critical int
 	}
 	UpdateLimiter float64
+	EmojiEnabled  bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	adminService admin
+	pubRest   public
+	privRest  private
+	adminRest admin
+	rssRest   rss
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
@@ -80,6 +85,7 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		err := s.httpServer.ListenAndServe()
@@ -89,7 +95,10 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		go func() {
@@ -106,7 +115,11 @@ func (s *Rest) Run(port int) {
 		m := s.makeAutocertManager()
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.lock.Unlock()
 
 		go func() {
@@ -149,24 +162,17 @@ func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 		Addr:              fmt.Sprintf(":%d", port),
 		Handler:           router,
 		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
+		// WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
-		IdleTimeout:       30 * time.Second,
+		IdleTimeout: 30 * time.Second,
 	}
 }
 
 func (s *Rest) routes() chi.Router {
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
 	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
 
-	s.adminService = admin{
+	s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
-		dataService:   s.DataService,
-		migrator:      s.Migrator,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
 
 	corsMiddleware := cors.New(cors.Options{
 		AllowedOrigins:   []string{"*"},
@@ -179,17 +185,19 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Flags(logger.All), logger.Log(log.Default()), logger.IPfn(ipFn), logger.Prefix("[INFO]"))
+		r.Use(middleware.Timeout(5 * time.Second))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
-		r.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+		r.Use(middleware.Timeout(5 * time.Second))
+		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
 
@@ -199,102 +207,232 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
-			rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(middleware.Timeout(5 * time.Second))
+			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
 		})
 
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
-			ropen.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
 			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
+			ropen.Get("/find", s.pubRest.findCommentsCtrl)
-			ropen.Get("/info", s.infoCtrl)
+			ropen.Get("/id/{id}", s.pubRest.commentByIDCtrl)
+			ropen.Get("/comments", s.pubRest.findUserCommentsCtrl)
+			ropen.Get("/last/{limit}", s.pubRest.lastCommentsCtrl)
+			ropen.Get("/count", s.pubRest.countCtrl)
+			ropen.Post("/counts", s.pubRest.countMultiCtrl)
+			ropen.Get("/list", s.pubRest.listCtrl)
+			ropen.Post("/preview", s.pubRest.previewCommentCtrl)
+			ropen.Get("/info", s.pubRest.infoCtrl)
+			ropen.Get("/img", s.ImageProxy.Handler)
 
-			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Route("/rss", func(rrss chi.Router) {
-			ropen.Mount("/img", s.ImageProxy.Routes())
+				rrss.Get("/post", s.rssRest.postCommentsCtrl)
+				rrss.Get("/site", s.rssRest.siteCommentsCtrl)
+				rrss.Get("/reply", s.rssRest.repliesCtrl)
+			})
+
+		})
+
+		// open routes, streams, no send timeout
+		rapi.Route("/stream", func(rstream chi.Router) {
+			rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
+			rstream.Get("/info", s.pubRest.infoStreamCtrl)
+			rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
+		})
+
+		// open routes, cached
+		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
+			ropen.Get("/picture/{user}/{id}", s.pubRest.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(middleware.Timeout(30 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(authMiddleware.Auth, middleware.NoCache, logInfoWithBody)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+			rauth.Get("/user", s.privRest.userInfoCtrl)
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			rauth.Get("/userdata", s.privRest.userAllDataCtrl)
-			rauth.Get("/user", s.userInfoCtrl)
+		})
-			rauth.Get("/userdata", s.userAllDataCtrl)
 
-			// admin routes, admin users only
+		// admin routes, require auth and admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(middleware.Timeout(30 * time.Second))
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminRest.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminRest.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminRest.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminRest.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminRest.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminRest.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminRest.setPinCtrl)
+			radmin.Get("/blocked", s.adminRest.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminRest.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminRest.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminRest.migrator.exportCtrl)
+			radmin.Post("/import", s.adminRest.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
+			radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
+			rauth.Use(middleware.Timeout(10 * time.Second))
-			if s.UpdateLimiter > 0 {
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
 			rauth.Use(authMiddleware.Auth)
-			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+			rauth.Use(middleware.NoCache)
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
+			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
-			rauth.Post("/comment", s.createCommentCtrl)
+			rauth.Post("/comment", s.privRest.createCommentCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
+			rauth.With(rejectAnonUser).Put("/vote/{id}", s.privRest.voteCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.privRest.deleteMeCtrl)
 		})
+
+		// protected routes, anonymous rejected
+		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(middleware.Timeout(10 * time.Second))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, rejectAnonUser)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Post("/picture", s.privRest.savePictureCtrl)
+		})
+
 	})
 
-	// respond to /robots.txt with the list of allowed paths
+	// open routes on root level
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
+	router.Group(func(rroot chi.Router) {
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		rroot.Use(middleware.Timeout(10 * time.Second))
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
-			for i := range allowed {
+		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
-				allowed[i] = "Allow: /api/v1" + allowed[i]
+		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
-			}
+	})
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
 
 	// file server for static content from /web
 	addFileServer(router, "/web", http.Dir(s.WebRoot))
 	return router
 }
 
+func (s *Rest) controllerGroups() (public, private, admin, rss) {
+
+	pubGrp := public{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		webRoot:          s.WebRoot,
+		streamer:         s.Streamer,
+	}
+
+	privGrp := private{
+		dataService:      s.DataService,
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		authenticator:    s.Authenticator,
+		notifyService:    s.NotifyService,
+		remarkURL:        s.RemarkURL,
+	}
+
+	admGrp := admin{
+		dataService:   s.DataService,
+		migrator:      s.Migrator,
+		cache:         s.Cache,
+		authenticator: s.Authenticator,
+		readOnlyAge:   s.ReadOnlyAge,
+	}
+
+	rssGrp := rss{
+		dataService: s.DataService,
+		cache:       s.Cache,
+	}
+
+	return pubGrp, privGrp, admGrp, rssGrp
+}
+
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
+// GET /config?site=siteID - returns configuration
+func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	admins, _ := s.DataService.AdminStore.Admins(siteID)
+	emails, _ := s.DataService.AdminStore.Email(siteID)
+
+	cnf := struct {
+		Version        string   `json:"version"`
+		EditDuration   int      `json:"edit_duration"`
+		MaxCommentSize int      `json:"max_comment_size"`
+		Admins         []string `json:"admins"`
+		AdminEmail     string   `json:"admin_email"`
+		Auth           []string `json:"auth_providers"`
+		LowScore       int      `json:"low_score"`
+		CriticalScore  int      `json:"critical_score"`
+		PositiveScore  bool     `json:"positive_score"`
+		ReadOnlyAge    int      `json:"readonly_age"`
+		MaxImageSize   int      `json:"max_image_size"`
+		EmojiEnabled   bool     `json:"emoji_enabled"`
+	}{
+		Version:        s.Version,
+		EditDuration:   int(s.DataService.EditDuration.Seconds()),
+		MaxCommentSize: s.DataService.MaxCommentSize,
+		Admins:         admins,
+		AdminEmail:     emails,
+		LowScore:       s.ScoreThresholds.Low,
+		CriticalScore:  s.ScoreThresholds.Critical,
+		PositiveScore:  s.DataService.PositiveScore,
+		ReadOnlyAge:    s.ReadOnlyAge,
+		MaxImageSize:   s.ImageService.Store.SizeLimit(),
+		EmojiEnabled:   s.EmojiEnabled,
+	}
+
+	cnf.Auth = []string{}
+	for _, ap := range s.Authenticator.Providers() {
+		cnf.Auth = append(cnf.Auth, ap.Name())
+	}
+
+	if cnf.Admins == nil { // prevent json serialization to nil
+		cnf.Admins = []string{}
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, cnf)
+}
+
 // serves static files from /web or embedded by statik
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 
 	var webFS http.Handler
 
 	statikFS, err := fs.New()
-	if err == nil {
-		log.Printf("[INFO] run file server for %s, embedded", root)
-		webFS = http.FileServer(statikFS)
-	}
 	if err != nil {
 		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
 		log.Printf("[INFO] run file server for %s, path %s", root, path)
 		webFS = http.FileServer(root)
+	} else {
+		log.Printf("[INFO] run file server for %s, embedded", root)
+		webFS = http.FileServer(statikFS)
 	}
 
 	origPath := path
@@ -305,15 +443,15 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	}
 	path += "*"
 
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)), middleware.Timeout(10*time.Second)).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Get(path, func(w http.ResponseWriter, r *http.Request) {
 			// don't show dirs, just serve files
 			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
 				http.NotFound(w, r)
 				return
 			}
 			webFS.ServeHTTP(w, r)
-		}))
+		})
 }
 
 func encodeJSONWithHTML(v interface{}) ([]byte, error) {
@@ -326,7 +464,8 @@ func encodeJSONWithHTML(v interface{}) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
+func filterComments(comments []store.Comment, fn func(c store.Comment) bool) []store.Comment {
+	filtered := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			filtered = append(filtered, c)
@@ -339,9 +478,67 @@ func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (fi
 // admins will have different keys in order to prevent leak of admin-only data to regular users
 func URLKey(r *http.Request) string {
 	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
+	if user, err := rest.GetUserInfo(r); err == nil && user.Admin {
-		key = adminPrefix + key
+		key = adminPrefix + key // make separate cache key for admins
 	}
 	return key
 }
+
+// URLKeyWithUser gets url from request to use it as cache key and attaching user ID
+// admins will have different keys in order to prevent leak of admin-only data to regular users
+func URLKeyWithUser(r *http.Request) string {
+	adminPrefix := "admin!!"
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix) // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil {
+		if user.Admin {
+			key = adminPrefix + user.ID + "!!" + key // make separate cache key for admins
+		} else {
+			key = user.ID + "!!" + key // make separate cache key for authed users
+		}
+	}
+	return key
+}
+
+// rejectAnonUser is a middleware rejecting anonymous users
+func rejectAnonUser(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		user, err := rest.GetUserInfo(r)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		if strings.HasPrefix(user.ID, "anonymous_") {
+			http.Error(w, "Access denied", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
+func parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private.go

@@ -9,26 +9,53 @@ import (
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
+type private struct {
+	dataService      privStore
+	cache            cache.LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	notifyService    *notify.Service
+	authenticator    *auth.Service
+	remarkURL        string
+}
+
+type privStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	EditComment(locator store.Locator, commentID string, req service.EditRequest) (comment store.Comment, err error)
+	Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	ValidateComment(c *store.Comment) error
+	IsVerified(siteID string, userID string) bool
+	IsReadOnly(locator store.Locator) bool
+	IsBlocked(siteID string, userID string) bool
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+}
+
 // POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -39,44 +66,44 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
 
 	comment.Orig = comment.Text // original comment text, prior to md render
-	if err := s.DataService.ValidateComment(&comment); err != nil {
+	if err := s.dataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 
 	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
+	if s.dataService.IsBlocked(comment.Locator.SiteID, comment.User.ID) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
 	if s.isReadOnly(comment.Locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
-	id, err := s.DataService.Create(comment)
+	id, err := s.dataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment", rest.ErrInternal)
 		return
 	}
 
-	// DataService modifies comment
+	// dataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
+	finalComment, err := s.dataService.Get(comment.Locator, id, rest.GetUserOrEmpty(r))
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+	s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
-	if s.NotifyService != nil {
+	if s.notifyService != nil {
-		s.NotifyService.Submit(finalComment)
+		s.notifyService.Submit(finalComment)
 	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -86,7 +113,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	edit := struct {
 		Text    string
@@ -95,7 +122,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
@@ -107,49 +134,52 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	var currComment store.Comment
 	var err error
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
+	if currComment, err = s.dataService.Get(locator, id, rest.GetUserOrEmpty(r)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
 
 	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"),
+			"can not edit comments for other users", rest.ErrNoAccess)
 		return
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.CommentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
 	}
 
-	res, err := s.DataService.EditComment(locator, id, editReq)
+	res, err := s.dataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
-		return
-	}
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope, user.ID))
+	if err != nil {
+		code := parseError(err, rest.ErrCommentRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
+		return
+	}
+
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
 // GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
+		user.Verified = s.dataService.IsVerified(siteID, user.ID)
 	}
 
 	render.JSON(w, r, user)
 }
 
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
@@ -158,32 +188,33 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	vote := r.URL.Query().Get("vote") == "1"
 
 	if s.isReadOnly(locator) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only", rest.ErrReadOnly)
 		return
 	}
 
 	// check if user blocked
-	if s.adminService.checkBlocked(locator.SiteID, user) {
+	if s.dataService.IsBlocked(locator.SiteID, user.ID) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
+	comment, err := s.dataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
+		code := parseError(err, rest.ErrVoteRejected)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
 	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info", rest.ErrInternal)
 		return
 	}
 
@@ -209,14 +240,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
+		comments, errUser := s.dataService.User(siteID, user.ID, 100, i*100, rest.GetUserOrEmpty(r))
-		if err != nil {
+		if errUser != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
+		b, errUser := json.Marshal(comments)
-		if err != nil {
+		if errUser != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
@@ -228,7 +259,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	merr = multierror.Append(merr, write([]byte(`}`)))
 	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info", rest.ErrInternal)
 		return
 	}
 
@@ -236,7 +267,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 // POST /deleteme?site_id=site - requesting delete of all user info
 // makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	siteID := r.URL.Query().Get("site")
 
@@ -256,22 +287,47 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
-	tokenStr, err := s.Authenticator.TokenService().Token(claims)
+	tokenStr, err := s.authenticator.TokenService().Token(claims)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
-	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
+	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.remarkURL, tokenStr)
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
-func (s *Rest) isReadOnly(locator store.Locator) bool {
+// POST /image - save image with form request
-	if s.ReadOnlyAge > 0 {
+func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	id, err := s.imageService.Save(header.Filename, user.ID, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"id": id})
+}
+
+func (s *private) isReadOnly(locator store.Locator) bool {
+	if s.readOnlyAge > 0 {
 		// check RO by age
-		if info, e := s.DataService.Info(locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+		if info, e := s.dataService.Info(locator, s.readOnlyAge); e == nil && info.ReadOnly {
 			return true
 		}
 	}
-	return s.DataService.IsReadOnly(locator) // ro manually
+	return s.dataService.IsReadOnly(locator) // ro manually
 }

backend/app/rest/api/rest_private_test.go

@@ -1,22 +1,34 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==`
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
 func TestRest_Create(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -48,7 +60,7 @@ func TestRest_CreateOldPost(t *testing.T) {
 	_, err := srv.DataService.Create(old)
 	assert.Nil(t, err)
 
-	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time")
+	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time", store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments))
 
@@ -216,21 +228,34 @@ func TestRest_UpdateDelete(t *testing.T) {
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	id := addComment(t, c1, ts)
 
+	// check multi count updated
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, b.StatusCode, string(body))
 
 	// comments returned by update
 	c2 := store.Comment{}
 	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, id, c2.ID)
 	assert.True(t, c2.Deleted)
 
@@ -243,6 +268,18 @@ func TestRest_UpdateDelete(t *testing.T) {
 	assert.Equal(t, "", c3.Text)
 	assert.Equal(t, "", c3.Orig)
 	assert.True(t, c3.Deleted)
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 }
 
 func TestRest_UpdateNotOwner(t *testing.T) {
@@ -264,7 +301,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	body, err := ioutil.ReadAll(b.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
+	assert.Equal(t, `{"code":3,"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
 
 	client = http.Client{}
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
@@ -318,7 +355,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("admin", "password")
+		req.Header.Add("X-JWT", devToken)
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -326,22 +363,65 @@ func TestRest_Vote(t *testing.T) {
 
 	assert.Equal(t, 200, vote(1), "first vote allowed")
 	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err := json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"admin": true}, cr.Votes)
+	assert.Equal(t, 1, cr.Vote)
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr = store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
+	assert.Equal(t, 0, cr.Vote)
+
+	assert.Equal(t, 200, vote(-1), "opposite vote allowed one more time")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	assert.Equal(t, 400, vote(-1), "dbl vote not allowed")
+	body, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, -1, cr.Vote)
+
+	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	assert.Equal(t, 200, code)
+	cr = store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for not authed user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	req, err := http.NewRequest("GET",
+		fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
+	assert.NoError(t, err)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	cr = store.Comment{}
+	err = json.NewDecoder(resp.Body).Decode(&cr)
+	assert.Nil(t, err)
+	assert.Equal(t, -1, cr.Score)
+	assert.Equal(t, 0, cr.Vote, "no vote info for different user")
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
+
+	assert.Equal(t, map[string]bool(nil), cr.Votes)
 }
 
 func TestRest_UserAllData(t *testing.T) {
@@ -407,13 +487,12 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
 
-	for i := 0; i < 478; i++ {
+	for i := 0; i < 51; i++ {
 		c.ID = fmt.Sprintf("id-%03d", i)
 		c.Timestamp = c.Timestamp.Add(time.Second)
 		_, err := srv.DataService.Create(c)
 		require.Nil(t, err)
 	}
-
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
@@ -429,7 +508,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
 		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
+	assert.Equal(t, 51, strings.Count(string(ungzBody), `"text":`), "51 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
@@ -464,3 +543,145 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	savePic := func(name string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 1462, len(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+}
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	imageService := svc.ImageService
+	imageService.Store = &image.FileSystem{
+		Staging:  "/tmp/remark42/images.staging",
+		Location: "/tmp/remark42/images",
+		MaxSize:  2000,
+	}
+	imageService.TTL = 100 * time.Millisecond
+
+	svc.privRest.imageService = imageService
+	svc.ImageService = imageService
+
+	dataService := svc.DataService
+	dataService.EditDuration = time.Millisecond * 100
+	dataService.ImageService = svc.ImageService
+	svc.privRest.dataService = dataService
+
+	uploadPicture := func(file string) (id string) {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, gopherPNG())
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.Contains(t, m["id"], ".png")
+		t.Logf(string(body))
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png")
+	id2 := uploadPicture("pic2.png")
+	id3 := uploadPicture("pic3.png")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}

backend/app/rest/api/rest_public.go

@@ -1,50 +1,98 @@
 package api
 
 import (
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/base64"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
+type public struct {
+	dataService      pubStore
+	cache            cache.LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	streamer         *Streamer
+	webRoot          string
+}
+
+type pubStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	UserCount(siteID, userID string) (int, error)
+	Count(locator store.Locator) (int, error)
+	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+
+	ValidateComment(c *store.Comment) error
+	IsReadOnly(locator store.Locator) bool
+	Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
+}
+
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
 	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
 		sort = "+" + sort[1:]
 	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
 
-	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	view := r.URL.Query().Get("view")
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	since, err := s.parseSince(r)
-		comments, e := s.DataService.Find(locator, sort)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
+		return
+	}
+	format := r.URL.Query().Get("format")
+	if format == "tree" {
+		since = time.Time{} // since doesn't make sense for tree
+	}
+
+	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v", locator, sort, format, since)
+
+	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
+	data, err := s.cache.Get(key, func() ([]byte, error) {
+		comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
 		if e != nil {
-			return nil, e
+			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.adminService.alterComments(comments, r)
+		comments = s.applyView(comments, view)
 		var b []byte
-		switch r.URL.Query().Get("format") {
+		switch format {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(comments, sort, s.readOnlyAge)
-			if s.DataService.IsReadOnly(locator) {
+			if tree.Nodes == nil { // eliminate json nil serialization
+				tree.Nodes = []*service.Node{}
+			}
+			if s.dataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
 			b, e = encodeJSONWithHTML(tree)
 		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
+			withInfo := commentsWithInfo{Comments: comments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
+			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
 				withInfo.Info = info
 			}
 			b, e = encodeJSONWithHTML(withInfo)
@@ -53,7 +101,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -63,37 +111,37 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
 		return
 	}
 
 	user, err := rest.GetUserInfo(r)
 	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
+		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info", rest.ErrNoAccess)
 		return
 	}
 	comment.User = user
 	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
+	if err = s.dataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 	comment.Sanitize()
 	render.HTML(w, r, comment.Text)
 }
 
 // GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
+		info, e := s.dataService.Info(locator, s.readOnlyAge)
 		if e != nil {
 			return nil, e
 		}
@@ -101,7 +149,7 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info", rest.ErrPostNotFound)
 		return
 	}
 
@@ -110,8 +158,53 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
+// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+	log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
+	fn := func() steamEventFn {
+		lastTS := sinceTs
+		lastCount := 0
+
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				info, e := s.dataService.Info(locator, s.readOnlyAge)
+				if e != nil {
+					return nil, e
+				}
+				// cache update used as indication of post update. comparing lastTS for no-cache.
+				// removal won't update lastTS, count check will catch it.
+				if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
+					upd = true
+				}
+				lastTS = info.LastTS
+				lastCount = info.Count
+				return encodeJSONWithHTML(info)
+			})
+			if err != nil {
+				return "info", data, false, err
+			}
+
+			return "info", data, upd, nil
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
+// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
+// limited with "since" param
+func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get last comments for %s", siteID)
 
@@ -120,20 +213,25 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		limit = 0
 	}
 
+	sinceTime, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
+		comments, e := s.dataService.Last(siteID, limit, sinceTime, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments", rest.ErrInternal)
 		return
 	}
 
@@ -142,8 +240,47 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
+func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] get last comments stream for %s", siteID)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+	if sinceTs.IsZero() {
+		sinceTs = time.Now()
+	}
+
+	fn := func() steamEventFn {
+		sinceTime := sinceTs
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
+				if e != nil {
+					return nil, e
+				}
+				if len(comments) > 0 {
+					sinceTime = comments[0].Timestamp
+					upd = true
+				}
+				sinceTime = time.Now()
+				return encodeJSONWithHTML(comments)
+			})
+			return "last", data, upd, err
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	id := chi.URLParam(r, "id")
 	siteID := r.URL.Query().Get("site")
@@ -151,12 +288,11 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
 
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
+	comment, err := s.dataService.Get(store.Locator{SiteID: siteID, URL: url}, id, rest.GetUserOrEmpty(r))
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -165,7 +301,7 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
@@ -182,15 +318,14 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
-	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID)
+	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
+		comments, e := s.dataService.User(siteID, userID, limit, 0, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
+		count, e := s.dataService.UserCount(siteID, userID)
 		if e != nil {
 			return nil, e
 		}
@@ -199,7 +334,7 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id", rest.ErrCommentNotFound)
 		return
 	}
 
@@ -208,76 +343,34 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.DataService.AdminStore.Admins(siteID),
-		AdminEmail:     s.DataService.AdminStore.Email(siteID),
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		ReadOnlyAge:    s.ReadOnlyAge,
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers() {
-		cnf.Auth = append(cnf.Auth, ap.Name())
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
 // GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
+	count, err := s.dataService.Count(locator)
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
 	}
 	render.JSON(w, r, R.JSON{"count": count, "locator": locator})
 }
 
-// POST /count?site=siteID - get number of comments for posts from post body
+// POST /counts?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
 		return
 	}
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
+	h := sha1.Sum([]byte(k)) // nolint
-	if _, err := hasher.Write([]byte(k)); err != nil {
+	sha := base64.URLEncoding.EncodeToString(h[:])
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
+
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
+		counts, e := s.dataService.Counts(siteID, posts)
 		if e != nil {
 			return nil, e
 		}
@@ -285,7 +378,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
@@ -295,7 +388,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 	siteID := r.URL.Query().Get("site")
 	limit, skip := 0, 0
@@ -308,8 +401,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
+		posts, e := s.dataService.List(siteID, limit, skip)
 		if e != nil {
 			return nil, e
 		}
@@ -317,11 +410,105 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID, rest.ErrSiteNotFound)
 		return
 	}
 
 	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
-		log.Printf("[WARN] can't render posts lits for site %s", siteID)
+		log.Printf("[WARN] can't render posts list for site %s", siteID)
 	}
 }
+
+// GET /picture/{user}/{id} - get picture
+func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
+	imgRdr, size, err := s.imageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer func() {
+		if e := imgRdr.Close(); e != nil {
+			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
+		}
+	}()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.webRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}
+
+func (s *public) applyView(comments []store.Comment, view string) []store.Comment {
+	if strings.EqualFold(view, "user") {
+		projection := make([]store.Comment, len(comments))
+		for i, c := range comments {
+			p := store.Comment{
+				ID:   c.ID,
+				User: c.User,
+			}
+			projection[i] = p
+		}
+		return projection
+	}
+	return comments
+}
+
+func (s *public) parseSince(r *http.Request) (time.Time, error) {
+	sinceTs := time.Time{}
+	if since := r.URL.Query().Get("since"); since != "" {
+		unixTS, e := strconv.ParseInt(since, 10, 64)
+		if e != nil {
+			return time.Time{}, errors.Wrap(e, "can't translate since parameter")
+		}
+		sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+	}
+	return sinceTs, nil
+}

backend/app/rest/api/rest_public_test.go

@@ -1,20 +1,25 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -36,6 +41,10 @@ func TestRest_Preview(t *testing.T) {
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
+
+	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
+	assert.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_PreviewWithMD(t *testing.T) {
@@ -69,8 +78,12 @@ func TestRest_Find(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -83,14 +96,16 @@ func TestRest_Find(t *testing.T) {
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
+	comments = commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
+	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "<p>test test #1</p>\n", comments.Comments[0].Text)
+	assert.Equal(t, "<p>test test #2</p>\n", comments.Comments[1].Text)
 	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
 	assert.Equal(t, 2, comments.Info.Count)
 	assert.Equal(t, false, comments.Info.ReadOnly)
@@ -106,7 +121,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -132,7 +147,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
@@ -173,7 +188,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -181,7 +196,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
+	tree = service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -190,21 +205,64 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.False(t, tree.Info.ReadOnly, "post is writable")
 }
 
+func TestRest_FindUserView(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&view=user")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id1 := addComment(t, c1, ts)
+
+	c2 := store.Comment{Text: "test test #2", ParentID: id1,
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id2 := addComment(t, c2, ts)
+
+	assert.NotEqual(t, id1, id2)
+
+	// get sorted by +time with view=user
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time&view=user")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, id1, comments.Comments[0].ID)
+	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[1].Text)
+}
+
 func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "[]\n", res, "empty last should return empty list")
+
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
 	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
+	ts1 := time.Now().UnixNano() / 1000000
 	addComment(t, c1, ts)
 	id1 := addComment(t, c1, ts)
+	time.Sleep(10 * time.Millisecond)
+	ts2 := time.Now().UnixNano() / 1000000
 	id2 := addComment(t, c2, ts)
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
@@ -213,6 +271,23 @@ func TestRest_Last(t *testing.T) {
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts1))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.Equal(t, id1, comments[1].ID)
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts2))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+	assert.Equal(t, id2, comments[0].ID)
+
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
@@ -227,12 +302,16 @@ func TestRest_Last(t *testing.T) {
 
 	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
 	assert.Nil(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
 	t.Logf("%+v", comments)
+
+	_, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t-BLAH")
+	assert.Equal(t, 500, code)
 }
 
 func TestRest_FindUserComments(t *testing.T) {
@@ -310,6 +389,9 @@ func TestRest_Count(t *testing.T) {
 	err = json.Unmarshal([]byte(body), &j)
 	assert.Nil(t, err)
 	assert.Equal(t, 2.0, j["count"])
+
+	_, code = get(t, ts.URL+"/api/v1/count?site=radio-t-BLAH&url=https://radio-t.com/blah1XXX")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_Counts(t *testing.T) {
@@ -339,6 +421,10 @@ func TestRest_Counts(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
 		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_List(t *testing.T) {
@@ -365,6 +451,40 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 2, pi[0].Count)
 	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
 	assert.Equal(t, 3, pi[1].Count)
+
+	_, code = get(t, ts.URL+"/api/v1/list?site=radio-t-BLAH")
+	assert.Equal(t, 400, code)
+}
+
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
 }
 
 func TestRest_Config(t *testing.T) {
@@ -382,7 +502,10 @@ func TestRest_Config(t *testing.T) {
 	assert.Equal(t, 4000., j["max_comment_size"])
 	assert.Equal(t, -5., j["low_score"])
 	assert.Equal(t, -10., j["critical_score"])
+	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
+	assert.Equal(t, 10000., j["max_image_size"])
+	assert.Equal(t, true, j["emoji_enabled"].(bool))
 	t.Logf("%+v", j)
 }
 
@@ -390,7 +513,7 @@ func TestRest_Info(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 
 	user := store.User{ID: "user1", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
@@ -423,6 +546,155 @@ func TestRest_Info(t *testing.T) {
 	assert.Equal(t, 400, code)
 }
 
+func TestRest_InfoStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	wg.Wait()
+
+	t.Logf(string(body))
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
+	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
+
+	_, code = get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah123")
+	assert.Equal(t, 500, code)
+}
+
+func TestRest_InfoStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+}
+
+func TestRest_InfoStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_InfoStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 5; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+			log.Printf("write #%d", i)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1", nil)
+	require.Nil(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	log.Print("start req")
+	r, err := client.Do(req)
+	log.Print("end req")
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 2*3, len(recs), "should have 2 events")
+	assert.True(t, strings.Contains(recs[0*3+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[1*3+1], `"count":3`), recs[1])
+}
+
+func TestRest_InfoStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1&since=12345678")
+	assert.Equal(t, 200, code)
+	wg.Wait()
+
+	t.Logf(string(body))
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
+}
+
 func TestRest_Robots(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -431,5 +703,179 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
+}
+
+func TestRest_LastCommentsStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	require.Nil(t, err)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+	t.Logf("headers: %+v", r.Header)
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 9*3, len(recs), "9 events")
+	t.Logf("%s", string(body))
+	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
+}
+
+func TestRest_LastCommentsStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_LastCommentsStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	require.Nil(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 2*3, len(recs), "2 events")
+	assert.True(t, strings.Contains(recs[0+1], `test 123`), recs[0+1])
+}
+
+func TestRest_LastCommentsStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	assert.Equal(t, 200, code, "all streams closed, good to go again")
+}
+
+func TestRest_LastCommentsStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t&since=123456", nil)
+	require.Nil(t, err)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+	t.Logf("headers: %+v", r.Header)
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 events, includes first record")
+	t.Logf("%v", recs)
+}
+
+func postComment(t *testing.T, url string) {
+	resp, e := post(t, url+"/api/v1/comment",
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	require.Nil(t, e)
+	b, e := ioutil.ReadAll(resp.Body)
+	require.Nil(t, e)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
 }

backend/app/rest/api/rest_test.go

@@ -4,10 +4,14 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -16,24 +20,35 @@ import (
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-var testDb = "/tmp/test-remark.db"
 var testHTML = "/tmp/test-remark.html"
 var getStartedHTML = "/tmp/getstarted.html"
 
-var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." +
+	"eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIs" +
+	"InVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMT" +
+	"I3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9." +
+	"eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1" +
+	"NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0" +
+	"dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlI" +
+	"iwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.gAR_sZT7hTx7CNHByyrJQWMB5tAtoiISAiG8kes1IjA"
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
@@ -175,18 +190,121 @@ func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
 	srv.Shutdown()
 }
 
+func TestRest_rejectAnonUser(t *testing.T) {
+
+	ts := httptest.NewServer(fakeAuth(rejectAnonUser(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "Hello")
+	}))))
+	defer ts.Close()
+
+	resp, err := http.Get(ts.URL)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "use not logged in")
+
+	resp, err = http.Get(ts.URL + "?fake_id=anonymous_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode, "anon rejected")
+
+	resp, err = http.Get(ts.URL + "?fake_id=real_user123&fake_name=test")
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "real user")
+}
+
+func Test_URLKey(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKey(r))
+		})
+	}
+
+}
+
+func Test_URLKeyWithUser(t *testing.T) {
+	tbl := []struct {
+		url  string
+		user store.User
+		key  string
+	}{
+		{"http://example.com/1", store.User{}, "http://example.com/1"},
+		{"http://example.com/1", store.User{ID: "user"}, "user!!http://example.com/1"},
+		{"http://example.com/2", store.User{ID: "user2"}, "user2!!http://example.com/2"},
+		{"http://example.com/1", store.User{ID: "user", Admin: true}, "admin!!user!!http://example.com/1"},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			r, err := http.NewRequest("GET", tt.url, nil)
+			require.NoError(t, err)
+			if tt.user.ID != "" {
+				r = rest.SetUserInfo(r, tt.user)
+			}
+			assert.Equal(t, tt.key, URLKeyWithUser(r))
+		})
+	}
+
+}
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}
+
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	// log.Setup(log.Out(ioutil.Discard))
+	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+
+	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
+	os.Remove(testDb)
+	os.Remove(testHTML)
+	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
+
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
 
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	memCache, err := cache.NewMemoryCache()
+	assert.NoError(t, err)
+
+	astore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
-		Interface:              b,
+		Engine:                 b,
 		EditDuration:           5 * time.Minute,
 		MaxCommentSize:         4000,
-		AdminStore:             adminStore,
+		AdminStore:             astore,
 		MaxVotes:               service.UnlimitedVotes,
 		RestrictedWordsMatcher: restrictedWordsMatcher,
 	}
@@ -198,10 +316,18 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
 			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
 		}),
-		Cache:     &cache.Nop{},
+		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+				Staging:    "/tmp/pics-remark42/staging",
+			},
+			TTL: time.Millisecond * 100,
+		},
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
@@ -211,8 +337,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
 			Cache:             &cache.Nop{},
-			KeyStore:          adminStore,
+			KeyStore:          astore,
 		},
+		Streamer: &Streamer{
+			Refresh:   100 * time.Millisecond,
+			TimeOut:   5 * time.Second,
+			MaxActive: 100,
+		},
+		EmojiEnabled: true,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
@@ -223,15 +355,30 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 
 	teardown = func() {
 		ts.Close()
-		srv.DataService.Close()
+		require.NoError(t, srv.DataService.Close())
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
 }
 
+// fake auth middleware make user authed and uses query's fake_id for ID and fake_name for Name
+func fakeAuth(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("fake_id") != "" {
+			r = rest.SetUserInfo(r, store.User{
+				ID:   r.URL.Query().Get("fake_id"),
+				Name: r.URL.Query().Get("fake_name"),
+			})
+		}
+		next.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(fn)
+}
+
 func get(t *testing.T, url string) (string, int) {
 	r, err := http.Get(url)
 	require.Nil(t, err)
@@ -241,6 +388,14 @@ func get(t *testing.T, url string) (string, int) {
 	return string(body), r.StatusCode
 }
 
+func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+	if token != "" {
+		r.Header.Set("X-JWT", token)
+	}
+	return client.Do(r)
+}
+
 func getWithDevAuth(t *testing.T, url string) (body string, code int) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
@@ -294,3 +449,13 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}

backend/app/rest/api/rss.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
@@ -15,135 +14,124 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
+type rss struct {
+	dataService rssStore
+	cache       cache.LoadingCache
+}
+
+type rssStore interface {
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error)
+}
+
 const maxRssItems = 20
-const maxLastCommentsReply = 1000
+const maxReplyDuration = 31 * 24 * time.Hour
-const maxReplyDuration = 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[DEBUG] get rss for post %+v", locator)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
+		comments, e := s.dataService.Find(locator, "-time", rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
+		feed, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
-		rss, e := s.toRssFeed(locator.URL, comments)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrPostNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
 // GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) siteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss for site %s", siteID)
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
+		comments, e := s.dataService.Last(siteID, maxRssItems, time.Time{}, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.adminService.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
+		feed, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
 // GET /rss/reply?user=userID&site=siteID
-func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) repliesCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() (res []byte, e error) {
+	data, err := s.cache.Get(key, func() (res []byte, e error) {
-		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+
+		replies, userName, e := s.dataService.UserReplies(siteID, userID, maxRssItems, maxReplyDuration)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.adminService.alterComments(comments, r)
-		replies := []store.Comment{}
-		for _, c := range comments {
-			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
-				break
-			}
-			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
-				var pc store.Comment
-				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
-					return nil, errors.Wrap(e, "can't get parent comment")
-				}
-				if pc.User.ID == userID {
-					replies = append(replies, c)
-				}
-			}
-		}
 
-		rss, e := s.toRssFeed(siteID, replies)
+		feed, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies")
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies", rest.ErrSiteNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
+func (s *rss) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
+	if description == "" {
+		description = "comment updates"
+	}
 	lastCommentTS := time.Unix(0, 0)
 	if len(comments) > 0 {
 		lastCommentTS = comments[0].Timestamp
@@ -152,7 +140,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 	feed := &feeds.Feed{
 		Title:       "Remark42 comments",
 		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
+		Description: description,
 		Created:     lastCommentTS,
 	}
 
@@ -168,9 +156,10 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
+			parentComment, err := s.dataService.Get(c.Locator, c.ParentID, store.User{})
 			if err == nil {
 				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
+				f.Description = f.Description + "<blockquote><p>" + parentComment.Snippet(300) + "</p></blockquote>"
 			} else {
 				log.Printf("[WARN] failed to get info about parent comment, %s", err)
 			}

backend/app/rest/api/rss_test.go

@@ -38,7 +38,7 @@ func TestServer_RssPost(t *testing.T) {
         <channel>
             <title>Remark42 comments</title>
             <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
+            <description>post comments for https://radio-t.com/blah1</description>
             <pubDate>%s</pubDate>
             <item>
 		      <title>developer one</title>
@@ -93,7 +93,7 @@ func TestServer_RssSite(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>radio-t</link>
-		    <description>comment updates</description>
+		    <description>site comment for radio-t</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one</title>
@@ -156,12 +156,12 @@ func TestServer_RssWithReply(t *testing.T) {
 		  <channel>
 		    <title>Remark42 comments</title>
 		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
+		    <description>post comments for https://radio-t.com/blah10</description>
 		    <pubDate>%s</pubDate>
 		    <item>
 		      <title>developer one &gt; developer one</title>
 		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
-		      <description>xyz test</description>
+              <description>xyz test&lt;blockquote&gt;&lt;p&gt;test 123&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>developer one</author>
               <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
@@ -242,12 +242,12 @@ func TestServer_RssReplies(t *testing.T) {
 	    <channel>
 	        <title>Remark42 comments</title>
 	        <link>radio-t</link>
-	        <description>comment updates</description>
+	        <description>replies to user1</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
-		      <description>reply to c1 from user3</description>
+		      <description>reply to c1 from user3&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user3</author>
               <guid>comment-3</guid> 
 		      <pubDate>%s</pubDate>
@@ -255,7 +255,7 @@ func TestServer_RssReplies(t *testing.T) {
 			<item>
 		      <title>user2 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
-		      <description>reply to c1 from user2</description>
+		      <description>reply to c1 from user2&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user2</author>
               <guid>comment-2</guid> 
 		      <pubDate>%s</pubDate>

backend/app/rest/api/ssl.go

@@ -86,7 +86,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager {
 // makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
 func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	cfg := makeTLSConfig()
+	cfg := s.makeTLSConfig()
 	cfg.GetCertificate = m.GetCertificate
 	server.TLSConfig = cfg
 	return server
@@ -95,7 +95,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocer
 // makeHTTPSServer makes https server for static mode
 func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	server.TLSConfig = makeTLSConfig()
+	server.TLSConfig = s.makeTLSConfig()
 	return server
 }
 
@@ -109,7 +109,7 @@ func (s *Rest) getRemarkHost() string {
 	return u.Hostname()
 }
 
-func makeTLSConfig() *tls.Config {
+func (s *Rest) makeTLSConfig() *tls.Config {
 	return &tls.Config{
 		PreferServerCipherSuites: true,
 		CipherSuites: []uint16{

backend/app/rest/api/stream.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// Streamer creates endless stream of \n separated json records send to remote client
+type Streamer struct {
+	TimeOut     time.Duration
+	Refresh     time.Duration
+	MaxActive   int32
+	activeCount int32
+}
+
+type steamEventFn func() (event string, data []byte, upd bool, err error)
+
+type steamEventResp struct {
+	data  []byte
+	event string
+	err   error
+}
+
+// Activate starts blocking function streaming update created by eventFn to ResponseWriter
+// canceled on context or inactivity timeout
+// note: eventFn is a closure needed to allow state management inside eventFn
+func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
+	updCh := s.eventsCh(ctx, eventFn())
+
+	count := atomic.AddInt32(&s.activeCount, 1)
+	defer atomic.AddInt32(&s.activeCount, -1)
+	if count > s.MaxActive {
+		return errors.New("too many streams")
+	}
+
+	if ww, ok := w.(http.ResponseWriter); ok {
+		ww.Header().Set("Content-Type", "text/event-stream")
+		ww.Header().Set("Connection", "keep-alive")
+		ww.Header().Set("Cache-Control", "no-cache")
+	}
+
+	for {
+		select {
+		case <-ctx.Done(): // request closed by remote client
+			log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
+			return nil
+		case <-time.After(s.TimeOut): // request closed by timeout
+			log.Printf("[DEBUG] stream closed due to timeout")
+			return nil
+		case resp, ok := <-updCh: // new update
+			if !ok { // closed updCh
+				return nil
+			}
+			if resp.err != nil {
+				return resp.err
+			}
+
+			// make server-sent event record
+			// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+			if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
+				return errors.Wrap(e, "send to stream failed")
+			}
+			if fw, okFlush := w.(http.Flusher); okFlush {
+				fw.Flush()
+			}
+		}
+	}
+}
+
+// populate updates to chan, break on context close
+func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
+	ch := make(chan steamEventResp)
+	go func() {
+		tick := time.NewTicker(s.Refresh)
+		defer func() {
+			close(ch)
+			tick.Stop()
+		}()
+		for {
+			select {
+			case <-ctx.Done(): // request closed by remote client
+				return
+			case <-tick.C:
+				event, resp, upd, err := fn()
+				if err != nil {
+					ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
+					return
+				}
+				if upd {
+					ch <- steamEventResp{event: event, data: resp, err: nil}
+				}
+			}
+		}
+	}()
+	return ch
+}

backend/app/rest/api/stream_test.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStream_Timeout(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 || n > 10 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	err := s.Activate(context.Background(), eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}
+
+func TestStream_Cancel(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err := s.Activate(ctx, eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}

backend/app/rest/httperrors.go

@@ -9,16 +9,40 @@ import (
 
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest"
+)
+
+// All error codes for UI mapping and translation
+const (
+	ErrInternal           = 0  // any internal error
+	ErrCommentNotFound    = 1  // can't find comment
+	ErrDecode             = 2  // failed to unmarshal incoming request
+	ErrNoAccess           = 3  // rejected by auth
+	ErrCommentValidation  = 4  // validation failed
+	ErrPostNotFound       = 5  // can't find post
+	ErrSiteNotFound       = 6  // can't find site
+	ErrUserBlocked        = 7  // user blocked
+	ErrReadOnly           = 8  // write failed on read only
+	ErrCommentRejected    = 9  // general error on rejected comment change
+	ErrCommentEditExpired = 10 // too late for edit
+	ErrCommentEditChanged = 11 // parent comment cannot be changed
+	ErrVoteRejected       = 12 // general error on vote rejected
+	ErrVoteSelf           = 13 // vote for own comment
+	ErrVoteDbl            = 14 // already voted for the comment
+	ErrVoteMax            = 15 // too many votes for the comment
+	ErrVoteMinScore       = 16 // min score reached for the comment
+	ErrActionRejected     = 17 // general error for rejected actions
+	ErrAssetNotFound      = 18 // requested file not found
 )
 
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
+func SendErrorJSON(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
+	log.Printf("[DEBUG] %s", errDetailsMsg(r, httpStatusCode, err, details, errCode))
-	render.Status(r, code)
+	render.Status(r, httpStatusCode)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
+	render.JSON(w, r, rest.JSON{"error": err.Error(), "details": details, "code": errCode})
 }
 
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
+func errDetailsMsg(r *http.Request, httpStatusCode int, err error, details string, errCode int) string {
 	uinfoStr := ""
 	if user, e := GetUserInfo(r); e == nil {
 		uinfoStr = user.Name + "/" + user.ID + " - "
@@ -40,5 +64,6 @@ func errDetailsMsg(r *http.Request, code int, err error, details string) string
 	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
 		remoteIP = remoteIP[:pos]
 	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
+	return fmt.Sprintf("%s - %v - %d (%d) - %s%s - %s%s",
+		details, err, httpStatusCode, errCode, uinfoStr, remoteIP, q, srcFileInfo)
 }

backend/app/rest/httperrors_test.go

@@ -17,7 +17,7 @@ func TestSendErrorJSON(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/error" {
 			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
+			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456", 123)
 			return
 		}
 		w.WriteHeader(404)
@@ -33,7 +33,7 @@ func TestSendErrorJSON(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 500, resp.StatusCode)
 
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
+	assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body))
 }
 
 func TestErrorDetailsMsg(t *testing.T) {
@@ -41,8 +41,8 @@ func TestErrorDetailsMsg(t *testing.T) {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
 		require.Nil(t, err)
 		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 123)
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (123) - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
 	}
 	callerFn()
 }
@@ -50,11 +50,13 @@ func TestErrorDetailsMsg(t *testing.T) {
 func TestErrorDetailsMsgWithUser(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
+		require.NoError(t, err)
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
 		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
+		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example." +
+			"com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:61 rest.TestErrorDetailsMsgWithUser]", msg)
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"context"
 	"encoding/base64"
 	"io"
 	"net/http"
@@ -8,7 +9,6 @@ import (
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
@@ -22,6 +22,7 @@ type Image struct {
 	RemarkURL string
 	RoutePath string
 	Enabled   bool
+	Timeout   time.Duration
 }
 
 // Convert all img src links without https to proxied links
@@ -38,64 +39,75 @@ func (p Image) Convert(commentHTML string) string {
 	return p.replace(commentHTML, imgs)
 }
 
-// Routes returns router group to respond to proxied request
+// Handler returns http handler respond to proxied request
-func (p Image) Routes() chi.Router {
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
-	router := chi.NewRouter()
+
 	if !p.Enabled {
-		return router
+		http.Error(w, "proxy disabled", http.StatusNotImplemented)
+		return
 	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
-			return
-		}
 
-		client := http.Client{Timeout: 30 * time.Second}
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		var resp *http.Response
+	if err != nil {
-		err = repeater.NewDefault(5, time.Second).Do(func() error {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-			var e error
+		return
-			resp, e = client.Get(string(src))
+	}
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
 
-		if resp.StatusCode != http.StatusOK {
+	timeout := 60 * time.Second // default
-			w.WriteHeader(resp.StatusCode)
+	if p.Timeout > 0 {
-			return
+		timeout = p.Timeout
-		}
+	}
 
-		for k, v := range resp.Header {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-			if strings.EqualFold(k, "Content-Type") {
+	defer cancel()
-				w.Header().Set(k, v[0])
+
-			}
+	client := http.Client{Timeout: 30 * time.Second}
-			if strings.EqualFold(k, "Content-Length") {
+	var resp *http.Response
-				w.Header().Set(k, v[0])
+	err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-			}
+		var e error
-		}
+		req, e := http.NewRequest("GET", string(src), nil)
-		// enforce client-side caching
+		if e != nil {
-		etag := `"` + r.URL.Query().Get("src") + `"`
+			return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
 		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
 	})
-	return router
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
+		return
+	}
+	defer func() {
+		if e := resp.Body.Close(); e != nil {
+			log.Printf("[WARN] can't close body, %s", e)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		w.WriteHeader(resp.StatusCode)
+		return
+	}
+
+	for k, v := range resp.Header {
+		if strings.EqualFold(k, "Content-Type") {
+			w.Header().Set(k, v[0])
+		}
+		if strings.EqualFold(k, "Content-Length") {
+			w.Header().Set(k, v[0])
+		}
+	}
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if _, e := io.Copy(w, resp.Body); e != nil {
+		log.Printf("[WARN] can't copy image stream, %s", e)
+	}
 }
 
 // extract gets all non-https images and return list of src

backend/app/rest/proxy/image_test.go

@@ -3,9 +3,13 @@ package proxy
 import (
 	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -45,9 +49,11 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{Enabled: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
-		assert.Nil(t, err, "err in #%d", i)
+			res, err := img.extract(tt.inp)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+			assert.Nil(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
@@ -60,12 +66,11 @@ func TestPicture_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
@@ -87,6 +92,24 @@ func TestImage_Routes(t *testing.T) {
 	assert.Equal(t, 400, resp.StatusCode)
 }
 
+func TestImage_RoutesTimedOut(t *testing.T) {
+	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
+
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
+	httpSrv := imgHTTPServer(t)
+	defer httpSrv.Close()
+
+	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
+	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
+	require.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	t.Log(string(b))
+	assert.True(t, strings.Contains(string(b), "deadline exceeded"))
+}
+
 func TestPicture_Convert(t *testing.T) {
 	img := Image{Enabled: true, RoutePath: "/img"}
 	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
@@ -110,7 +133,13 @@ func imgHTTPServer(t *testing.T) *httptest.Server {
 			t.Log("http img request", r.URL)
 			w.Header().Add("Content-Length", "123")
 			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			assert.NoError(t, err)
+			return
+		}
+		if r.URL.Path == "/image/img-slow.png" {
+			time.Sleep(500 * time.Millisecond)
+			w.WriteHeader(500)
 			return
 		}
 		t.Log("http img request - not found", r.URL)

backend/app/rest/user.go

@@ -36,7 +36,15 @@ func GetUserInfo(r *http.Request) (user store.User, err error) {
 		Verified: u.BoolAttr("verified"),
 		Blocked:  u.BoolAttr("blocked"),
 	}, nil
+}
 
+// GetUserOrEmpty attempts to get user info from request and returns empty object if failed
+func GetUserOrEmpty(r *http.Request) store.User {
+	user, err := GetUserInfo(r)
+	if err != nil {
+		return store.User{}
+	}
+	return user
 }
 
 // SetUserInfo sets user into request context

backend/app/rpc/client.go

@@ -0,0 +1,74 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+)
+
+// Client implements remote engine and delegates all calls to remote http server
+type Client struct {
+	API        string
+	Client     http.Client
+	AuthUser   string
+	AuthPasswd string
+
+	id uint64
+}
+
+// Call remote server with given method and arguments
+func (r *Client) Call(method string, args ...interface{}) (*Response, error) {
+
+	var b []byte
+	var err error
+
+	switch {
+	case len(args) == 0:
+		b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	case len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct:
+		b, err = json.Marshal(Request{Method: method, Params: args[0], ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	default:
+		b, err = json.Marshal(Request{Method: method, Params: args, ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	}
+
+	req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to make request for %s", method)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	if r.AuthUser != "" && r.AuthPasswd != "" {
+		req.SetBasicAuth(r.AuthUser, r.AuthPasswd)
+	}
+	resp, err := r.Client.Do(req)
+	if err != nil {
+		return nil, errors.Wrapf(err, "remote call failed for %s", method)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, errors.Errorf("bad status %s for %s", resp.Status, method)
+	}
+
+	cr := Response{}
+	if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil {
+		return nil, errors.Wrapf(err, "failed to decode response for %s", method)
+	}
+
+	if cr.Error != "" {
+		return nil, errors.New(cr.Error)
+	}
+	return &cr, nil
+}

backend/app/rpc/client_test.go

@@ -0,0 +1,97 @@
+package rpc
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClient_Call(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	resp, err := c.Call("test", 123, "abc")
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithObject(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":{"F1":123,"F2":"abc","F3":"2019-06-09T23:03:55Z"},"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	obj := struct {
+		F1 int
+		F2 string
+		F3 time.Time
+	}{
+		F1: 123,
+		F2: "abc",
+		F3: time.Date(2019, 6, 9, 23, 3, 55, 0, time.UTC),
+	}
+
+	resp, err := c.Call("test", obj)
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithNoParams(t *testing.T) {
+	ts := testServer(t, `{"method":"test","id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	resp, err := c.Call("test")
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallError(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	_, err := c.Call("test", 123, "abc")
+	assert.EqualError(t, err, "some error")
+}
+
+func TestClient_CallBadResponse(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345 invalid}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	_, err := c.Call("test", 123, "abc")
+	assert.NotNil(t, err)
+}
+
+func TestClient_CallBadRemote(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}
+	_, err := c.Call("test", 123)
+	assert.NotNil(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/rpc/remote.go

@@ -0,0 +1,23 @@
+// Package rpc implements client ans server for RPC-like communication with remote storage.
+// The protocol is somewhat simplified version of json-rpc with a single POST call sending
+// Request json (method name and the list of parameters) and receiving back json Response with "result" json
+// and error string
+package rpc
+
+import (
+	"encoding/json"
+)
+
+// Request encloses method name and all params
+type Request struct {
+	Method string      `json:"method"`
+	Params interface{} `json:"params,omitempty"`
+	ID     uint64      `json:"id"`
+}
+
+// Response encloses result and error received from remote server
+type Response struct {
+	Result *json.RawMessage `json:"result,omitempty"`
+	Error  string           `json:"error,omitempty"`
+	ID     uint64           `json:"id"`
+}

backend/app/rpc/server.go

@@ -0,0 +1,179 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/didip/tollbooth"
+	"github.com/didip/tollbooth_chi"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/render"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/logger"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/rest"
+)
+
+// Server is json-rpc server with an optional basic auth
+type Server struct {
+	API        string
+	AuthUser   string
+	AuthPasswd string
+	Version    string
+	AppName    string
+
+	funcs struct {
+		m    map[string]ServerFn
+		once sync.Once
+	}
+
+	httpServer struct {
+		*http.Server
+		sync.Mutex
+	}
+}
+
+// Encoder is a function to encode call's result to Response
+type Encoder func(id uint64, resp interface{}, e error) (Response, error)
+
+// ServerFn handler registered for each method with Add
+// Implementations provided by consumer and define response logic.
+type ServerFn func(id uint64, params json.RawMessage) Response
+
+// Run http server on given port
+func (s *Server) Run(port int) error {
+	if s.AuthUser == "" || s.AuthPasswd == "" {
+		log.Print("[WARN] extension server runs without auth")
+	}
+	if s.funcs.m == nil && len(s.funcs.m) == 0 {
+		return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run")
+	}
+
+	router := chi.NewRouter()
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(R.AppInfo(s.AppName, "umputun", s.Version), R.Ping)
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler
+	router.Use(middleware.Timeout(5 * time.Second))
+	router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(1000, nil)), middleware.NoCache)
+	router.Use(s.basicAuth)
+
+	router.Post(s.API, s.handler)
+
+	s.httpServer.Lock()
+	s.httpServer.Server = &http.Server{
+		Addr:              fmt.Sprintf(":%d", port),
+		Handler:           router,
+		ReadHeaderTimeout: 5 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       30 * time.Second,
+	}
+	s.httpServer.Unlock()
+
+	log.Printf("[INFO] listen on %d", port)
+	return s.httpServer.ListenAndServe()
+}
+
+// EncodeResponse convert anything to Response
+func (s *Server) EncodeResponse(id uint64, resp interface{}, e error) (Response, error) {
+	v, err := json.Marshal(&resp)
+	if err != nil {
+		return Response{}, err
+	}
+	if e != nil {
+		return Response{ID: id, Result: nil, Error: e.Error()}, nil
+	}
+	raw := json.RawMessage{}
+	if err = raw.UnmarshalJSON(v); err != nil {
+		return Response{}, err
+	}
+	return Response{ID: id, Result: &raw}, nil
+}
+
+// Shutdown http server
+func (s *Server) Shutdown() error {
+	s.httpServer.Lock()
+	defer s.httpServer.Unlock()
+	if s.httpServer.Server == nil {
+		return errors.Errorf("http server is not running")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return s.httpServer.Shutdown(ctx)
+}
+
+// Add method handler
+func (s *Server) Add(method string, fn ServerFn) {
+	s.httpServer.Lock()
+	defer s.httpServer.Unlock()
+	if s.httpServer.Server != nil {
+		log.Printf("[WARN] ignored method %s, can't be added to activated server", method)
+		return
+	}
+
+	s.funcs.once.Do(func() {
+		s.funcs.m = map[string]ServerFn{}
+	})
+
+	s.funcs.m[method] = fn
+	log.Printf("[INFO] add handler for %s", method)
+}
+
+// HandlersGroup alias for map of handlers
+type HandlersGroup map[string]ServerFn
+
+// Group of handlers with common prefix
+func (s *Server) Group(prefix string, m HandlersGroup) {
+	for k, v := range m {
+		s.Add(prefix+"."+k, v)
+	}
+}
+
+func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
+	req := struct {
+		ID     uint64           `json:"id"`
+		Method string           `json:"method"`
+		Params *json.RawMessage `json:"params"`
+	}{}
+
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, req.Method, 0)
+		return
+	}
+	fn, ok := s.funcs.m[req.Method]
+	if !ok {
+		rest.SendErrorJSON(w, r, http.StatusNotImplemented, errors.New("unsupported method"), req.Method, 0)
+		return
+	}
+
+	params := json.RawMessage{}
+	if req.Params != nil {
+		params = *req.Params
+	}
+
+	render.JSON(w, r, fn(req.ID, params))
+}
+
+func (s *Server) basicAuth(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if s.AuthUser == "" || s.AuthPasswd == "" {
+			h.ServeHTTP(w, r)
+			return
+		}
+
+		user, pass, ok := r.BasicAuth()
+		if user != s.AuthUser || pass != s.AuthPasswd || !ok {
+			w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+		h.ServeHTTP(w, r)
+	})
+}

backend/app/rpc/server_test.go

@@ -0,0 +1,245 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServerPrimitiveTypes(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+
+	type respData struct {
+		Res1 string
+		Res2 bool
+	}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	// check with direct http call
+	clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true}, ID: 123}
+	b := bytes.Buffer{}
+	require.NoError(t, json.NewEncoder(&b).Encode(clientReq))
+	resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	data, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true},"id":123}`+"\n", string(data))
+
+	// check with client call
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	r, err := c.Call("test", "blah", 42, true)
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+
+	res := respData{}
+	err = json.Unmarshal(*r.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+	assert.Equal(t, uint64(1), r.ID)
+}
+
+func TestServerWithObject(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+
+	type respData struct {
+		Res1 string
+		Res2 bool
+	}
+
+	type reqData struct {
+		Time time.Time
+		F1   string
+		F2   time.Duration
+	}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		arg := reqData{}
+		if err := json.Unmarshal(params, &arg); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", arg)
+
+		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	r, err := c.Call("test", reqData{Time: time.Now(), F1: "sawert", F2: time.Minute})
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+
+	res := respData{}
+	err = json.Unmarshal(*r.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+}
+
+func TestServerMethodNotImplemented(t *testing.T) {
+	s := Server{}
+	ts := httptest.NewServer(http.HandlerFunc(s.handler))
+	defer ts.Close()
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+
+	r := Request{Method: "blah"}
+	buf := bytes.Buffer{}
+	assert.NoError(t, json.NewEncoder(&buf).Encode(r))
+	resp, err := http.Post(ts.URL, "application/json", &buf)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNotImplemented, resp.StatusCode)
+
+	assert.EqualError(t, s.Shutdown(), "http server is not running")
+}
+
+func TestServerWithAuth(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, "res blah", nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	time.Sleep(10 * time.Millisecond)
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+	r, err := c.Call("test", "blah", 42, true)
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+	val := ""
+	err = json.Unmarshal(*r.Result, &val)
+	assert.NoError(t, err)
+	assert.Equal(t, "res blah", val)
+
+	c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err = c.Call("test", "blah", 42, true)
+	assert.EqualError(t, err, "bad status 401 Unauthorized for test")
+}
+
+func TestServerErrReturn(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, "res blah", errors.New("some error"))
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+	_, err := c.Call("test", "blah", 42, true)
+	assert.EqualError(t, err, "some error")
+}
+
+func TestServerGroup(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+	s.Group("pre", HandlersGroup{
+		"fn1": func(id uint64, params json.RawMessage) Response {
+			return Response{}
+		},
+		"fn2": func(id uint64, params json.RawMessage) Response {
+			return Response{}
+		},
+	})
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err := c.Call("fn1")
+	assert.EqualError(t, err, "bad status 501 Not Implemented for fn1")
+
+	_, err = c.Call("pre.fn1")
+	assert.NoError(t, err)
+	_, err = c.Call("pre.fn2")
+	assert.NoError(t, err)
+}
+
+func TestServerAddLate(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+	s.Add("fn1", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	// too late, ignored after run
+	s.Add("fn2", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err := c.Call("fn1")
+	assert.NoError(t, err)
+	_, err = c.Call("fn2")
+	assert.EqualError(t, err, "bad status 501 Not Implemented for fn2")
+}
+
+func TestServerNoHandlers(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+	assert.EqualError(t, s.Run(9091), "nothing mapped for dispatch, Add has to be called prior to Run")
+}

backend/app/store/admin/admin.go

@@ -10,25 +10,17 @@ import (
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key() (key string, err error)
-	Admins(siteID string) (ids []string)
+	Admins(siteID string) (ids []string, err error)
-	Email(siteID string) (email string)
+	Email(siteID string) (email string, err error)
 }
 
-// StaticStore implements keys.Store with a single, predefined key
+// StaticStore implements keys.Store with a single set of admins and email for all sites
 type StaticStore struct {
 	admins []string
 	email  string
 	key    string
 }
 
-// Key returns static key for all sites, allows empty site
-func (s *StaticStore) Key() (key string, err error) {
-	if s.key == "" {
-		return "", errors.New("empty key for static key store")
-	}
-	return s.key, nil
-}
-
 // NewStaticStore makes StaticStore instance with given key
 func NewStaticStore(key string, admins []string, email string) *StaticStore {
 	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
@@ -40,12 +32,20 @@ func NewStaticKeyStore(key string) *StaticStore {
 	return &StaticStore{key: key, admins: []string{}, email: ""}
 }
 
+// Key returns static key, same for all sites
+func (s *StaticStore) Key() (key string, err error) {
+	if s.key == "" {
+		return "", errors.New("empty key for static key store")
+	}
+	return s.key, nil
+}
+
 // Admins returns static list of admin's ids, the same for all sites
-func (s *StaticStore) Admins(string) (ids []string) {
+func (s *StaticStore) Admins(string) (ids []string, err error) {
-	return s.admins
+	return s.admins, nil
 }
 
 // Email gets static email address
-func (s *StaticStore) Email(string) (email string) {
+func (s *StaticStore) Email(string) (email string, err error) {
-	return s.email
+	return s.email, nil
 }

backend/app/store/admin/admin_test.go

@@ -3,10 +3,7 @@ package admin
 import (
 	"testing"
 
-	"github.com/globalsign/mgo"
-	"github.com/go-pkgz/mongo"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestStaticStore_Get(t *testing.T) {
@@ -16,48 +13,11 @@ func TestStaticStore_Get(t *testing.T) {
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 
-	a := ks.Admins("any")
+	a, err := ks.Admins("any")
+	assert.NoError(t, err)
 	assert.Equal(t, []string{"123", "xyz"}, a)
 
-	email := ks.Email("blah")
+	email, err := ks.Email("blah")
+	assert.NoError(t, err)
 	assert.Equal(t, "aa@example.com", email)
 }
-
-func TestMongoStore_Get(t *testing.T) {
-	conn, err := mongo.MakeTestConnection(t)
-	require.NoError(t, err)
-	var ms Store = NewMongoStore(conn, "secret")
-
-	recs := []mongoRec{
-		{"site1", []string{"i11", "i12"}, "e1"},
-		{"site2", []string{"i21", "i22"}, "e2"},
-	}
-	err = conn.WithCollection(func(coll *mgo.Collection) error {
-		if e1 := coll.Insert(recs[0]); e1 != nil {
-			return e1
-		}
-		return coll.Insert(recs[1])
-	})
-	require.NoError(t, err)
-
-	admins := ms.Admins("site1")
-	assert.Equal(t, []string{"i11", "i12"}, admins)
-	email := ms.Email("site1")
-	assert.Equal(t, "e1", email)
-	key, err := ms.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("site2")
-	assert.Equal(t, []string{"i21", "i22"}, admins)
-	email = ms.Email("site2")
-	assert.Equal(t, "e2", email)
-	key, err = ms.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("no-site-in-db")
-	assert.Equal(t, []string{}, admins)
-	email = ms.Email("no-site-in-db")
-	assert.Equal(t, "", email)
-}

backend/app/store/admin/mongo.go

@@ -1,56 +0,0 @@
-package admin
-
-import (
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	log "github.com/go-pkgz/lgr"
-
-	"github.com/go-pkgz/mongo"
-)
-
-// MongoStore implements admin.Store with mongo backend
-type MongoStore struct {
-	connection *mongo.Connection
-	key        string
-}
-
-type mongoRec struct {
-	SiteID string   `bson:"site"`
-	IDs    []string `bson:"admin_ids"`
-	Email  string   `bson:"admin_email"`
-}
-
-// NewMongoStore makes admin Store for mongo's connection
-func NewMongoStore(conn *mongo.Connection, key string) *MongoStore {
-	log.Printf("[DEBUG] make mongo admin store with %+v", conn)
-	return &MongoStore{connection: conn, key: key}
-}
-
-// Key executes find by siteID and returns substructure with secret key
-func (m *MongoStore) Key() (key string, err error) {
-	return m.key, nil
-}
-
-// Admins executes find by siteID and returns admins ids
-func (m *MongoStore) Admins(siteID string) (ids []string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return []string{}
-	}
-	return resp.IDs
-}
-
-// Email executes find by siteID and returns admin's email
-func (m *MongoStore) Email(siteID string) (email string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return ""
-	}
-	return resp.Email
-}

backend/app/store/admin/remote.go

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"encoding/json"
+
+	"github.com/umputun/remark/backend/app/rpc"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	rpc.Client
+}
+
+// Key returns the key, same for all sites
+func (r *RPC) Key() (key string, err error) {
+	resp, err := r.Call("admin.key")
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &key)
+	return key, err
+}
+
+// Admins returns list of admin's ids for given site
+func (r *RPC) Admins(siteID string) (ids []string, err error) {
+	resp, err := r.Call("admin.admins", siteID)
+	if err != nil {
+		return []string{}, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+		return []string{}, err
+	}
+	return ids, nil
+}
+
+// Email gets email address for given site
+func (r *RPC) Email(siteID string) (email string, err error) {
+	resp, err := r.Call("admin.email", siteID)
+	if err != nil {
+		return "", err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &email); err != nil {
+		return "", err
+	}
+	return email, nil
+}

backend/app/store/admin/remote_test.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/rpc"
+)
+
+func TestRemote_Key(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.key","id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Admins(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.admins","params":["site-1"],"id":1}`,
+		`{"result":["id1","id2"],"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Admins("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Email(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.email","params":["site-1"],"id":1}`,
+		`{"result":"bbb@example.com","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Email("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, "bbb@example.com", res)
+	t.Logf("%v %T", res, res)
+}
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/store/comment.go

@@ -3,6 +3,7 @@ package store
 import (
 	"html/template"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/microcosm-cc/bluemonday"
@@ -10,19 +11,21 @@ import (
 
 // Comment represents a single comment with optional reference to its parent
 type Comment struct {
-	ID        string          `json:"id" bson:"_id"`
+	ID          string          `json:"id" bson:"_id"`
-	ParentID  string          `json:"pid"`
+	ParentID    string          `json:"pid"`
-	Text      string          `json:"text"`
+	Text        string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
+	Orig        string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
+	User        User            `json:"user"`
-	Locator   Locator         `json:"locator"`
+	Locator     Locator         `json:"locator"`
-	Score     int             `json:"score"`
+	Score       int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
+	Votes       map[string]bool `json:"votes,omitempty"`
-	Timestamp time.Time       `json:"time" bson:"time"`
+	Vote        int             `json:"vote"` // vote for the current user, -1/1/0.
-	Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+	Controversy float64         `json:"controversy,omitempty"`
-	Pin       bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Timestamp   time.Time       `json:"time" bson:"time"`
-	Deleted   bool            `json:"delete,omitempty" bson:"delete"`
+	Edit        *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
-	PostTitle string          `json:"title,omitempty" bson:"title"`
+	Pin         bool            `json:"pin,omitempty" bson:"pin,omitempty"`
+	Deleted     bool            `json:"delete,omitempty" bson:"delete"`
+	PostTitle   string          `json:"title,omitempty" bson:"title"`
 }
 
 // Locator keeps site and url of the post
@@ -64,6 +67,7 @@ const (
 
 // Maximum length for URL text shortening.
 const shortURLLen = 48
+const snippetLen = 200
 
 // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
 // autogen fields and reset everything users not supposed to provide
@@ -105,3 +109,24 @@ func (c *Comment) Sanitize() {
 	c.User.Name = template.HTMLEscapeString(c.User.Name)
 	c.User.Picture = p.Sanitize(c.User.Picture)
 }
+
+// Snippet from comment's text
+func (c *Comment) Snippet(limit int) string {
+	if limit <= 0 {
+		limit = snippetLen
+	}
+	cleanText := strings.Replace(c.Text, "\n", " ", -1)
+	size := len([]rune(cleanText))
+	if size < limit {
+		return cleanText
+	}
+	snippet := []rune(cleanText)[:size]
+	// go back in snippet and found the first space
+	for i := len(snippet) - 1; i >= 0; i-- {
+		if snippet[i] == ' ' {
+			snippet = snippet[:i]
+			break
+		}
+	}
+	return string(snippet) + " ..."
+}

backend/app/store/comment_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -34,6 +35,14 @@ func TestComment_Sanitize(t *testing.T) {
 				User: User{ID: "id", Name: "xyz"},
 			},
 		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -120,3 +129,25 @@ func TestComment_SetDeletedHard(t *testing.T) {
 	assert.False(t, comment.Pin)
 	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
 }
+
+func TestComment_Snippet(t *testing.T) {
+	tbl := []struct {
+		limit int
+		inp   string
+		out   string
+	}{
+		{0, "", ""},
+		{-1, "test\nblah", "test blah"},
+		{5, "test\nblah", "test ..."},
+		{5, "xyz12345 xxx", "xyz12345 ..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			c := Comment{Text: tt.inp}
+			out := c.Snippet(tt.limit)
+			assert.Equal(t, tt.out, out)
+		})
+	}
+}

backend/app/store/engine/bolt.go

@@ -0,0 +1,866 @@
+package engine
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
+// there are 5 types of top-level buckets:
+//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
+//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
+//    value is not full comment but a reference combined from post-url+commentID
+//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
+//    is a nested bucket named userID with kv as ts:reference
+//  - blocking info sits in "block" bucket. Key is userID, value - ts
+//  - counts per post to keep number of comments. Key is post url, value - count
+//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
+type BoltDB struct {
+	dbs map[string]*bolt.DB
+}
+
+const (
+	// top level buckets
+	postsBucketName    = "posts"
+	lastBucketName     = "last"
+	userBucketName     = "users"
+	blocksBucketName   = "block"
+	infoBucketName     = "info"
+	readonlyBucketName = "readonly"
+	verifiedBucketName = "verified"
+
+	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
+)
+
+// BoltSite defines single site param
+type BoltSite struct {
+	FileName string // full path to boltdb
+	SiteID   string // ID of given site
+}
+
+// NewBoltDB makes persistent boltdb-based store. For each site new boltdb file created
+func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
+	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
+	result := BoltDB{dbs: make(map[string]*bolt.DB)}
+	for _, site := range sites {
+		db, err := bolt.Open(site.FileName, 0600, &options)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
+		}
+
+		// make top-level buckets
+		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
+			readonlyBucketName, verifiedBucketName}
+		err = db.Update(func(tx *bolt.Tx) error {
+			for _, bktName := range topBuckets {
+				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+				}
+			}
+			return nil
+		})
+
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create top level bucket)")
+		}
+
+		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
+	}
+	return &result, nil
+}
+
+// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
+func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
+
+	bdb, err := b.db(comment.Locator.SiteID)
+	if err != nil {
+		return "", err
+	}
+
+	if b.checkFlag(FlagRequest{Locator: comment.Locator, Flag: ReadOnly}) {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	err = bdb.Update(func(tx *bolt.Tx) (err error) {
+		var postBkt, lastBkt, userBkt *bolt.Bucket
+
+		if postBkt, err = b.makePostBucket(tx, comment.Locator.URL); err != nil {
+			return err
+		}
+		// check if key already in store, reject doubles
+		if postBkt.Get([]byte(comment.ID)) != nil {
+			return errors.Errorf("key %s already in store", comment.ID)
+		}
+
+		// serialize comment to json []byte for bolt and save
+		if err = b.save(postBkt, comment.ID, comment); err != nil {
+			return errors.Wrapf(err, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
+		}
+
+		ref := b.makeRef(comment) // reference combines url and comment id
+
+		// add reference to comment to "last" bucket
+		lastBkt = tx.Bucket([]byte(lastBucketName))
+		commentTs := []byte(comment.Timestamp.Format(tsNano))
+		if err = lastBkt.Put(commentTs, ref); err != nil {
+			return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
+		}
+
+		// add reference to commentID to "users" bucket
+		if userBkt, err = b.getUserBucket(tx, comment.User.ID); err != nil {
+			return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
+		}
+		// put into individual user's bucket with ts as a key
+		if err = userBkt.Put(commentTs, ref); err != nil {
+			return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
+		}
+
+		// set info with the count for post url
+		if _, err = b.setInfo(tx, comment); err != nil {
+			return errors.Wrapf(err, "failed to set info for %s", comment.Locator)
+		}
+		return nil
+	})
+
+	return comment.ID, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (b *BoltDB) Get(req GetRequest) (comment store.Comment, err error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return comment, err
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		bucket, e := b.getPostBucket(tx, req.Locator.URL)
+		if e != nil {
+			return e
+		}
+		return b.load(bucket, req.CommentID, &comment)
+	})
+	return comment, err
+}
+
+// Find returns all comments for given request and sorts results
+func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
+	comments = []store.Comment{}
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find post comments, i.e. for site and url
+		err = bdb.View(func(tx *bolt.Tx) error {
+
+			bucket, e := b.getPostBucket(tx, req.Locator.URL)
+			if e != nil {
+				return e
+			}
+
+			return bucket.ForEach(func(k, v []byte) error {
+				comment := store.Comment{}
+				if e = json.Unmarshal(v, &comment); e != nil {
+					return errors.Wrap(e, "failed to unmarshal")
+				}
+				if req.Since.IsZero() || comment.Timestamp.After(req.Since) {
+					comments = append(comments, comment)
+				}
+				return nil
+			})
+		})
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		comments, err = b.lastComments(req.Locator.SiteID, req.Limit, req.Since)
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments, err = b.userComments(req.Locator.SiteID, req.UserID, req.Limit, req.Skip)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	return SortComments(comments, req.Sort), nil
+}
+
+// Flag sets and gets flag values
+func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
+	if req.Update == FlagNonSet { // read flag value, no update requested
+		return b.checkFlag(req), nil
+	}
+
+	// write flag value
+	return b.setFlag(req)
+}
+
+// Update for locator.URL with mutable part of comment
+func (b *BoltDB) Update(comment store.Comment) error {
+
+	getReq := GetRequest{Locator: comment.Locator, CommentID: comment.ID}
+	if curComment, err := b.Get(getReq); err == nil {
+		// preserve immutable fields
+		comment.ParentID = curComment.ParentID
+		comment.Locator = curComment.Locator
+		comment.Timestamp = curComment.Timestamp
+		comment.User = curComment.User
+	}
+
+	bdb, err := b.db(comment.Locator.SiteID)
+	if err != nil {
+		return err
+	}
+
+	return bdb.Update(func(tx *bolt.Tx) error {
+		bucket, e := b.getPostBucket(tx, comment.Locator.URL)
+		if e != nil {
+			return e
+		}
+		return b.save(bucket, comment.ID, comment)
+	})
+}
+
+// Count returns number of comments for post or user
+func (b *BoltDB) Count(req FindRequest) (count int, err error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return 0, err
+	}
+
+	if req.Locator.URL != "" { // comment's count for post
+		err = bdb.View(func(tx *bolt.Tx) error {
+			var e error
+			count, e = b.count(tx, req.Locator.URL, 0)
+			return e
+		})
+		return count, err
+	}
+
+	if req.UserID != "" { // comment's count for user
+		err = bdb.View(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(userBucketName))
+			userIDBkt := usersBkt.Bucket([]byte(req.UserID))
+			if userIDBkt == nil {
+				return errors.Errorf("no comments for user %s in store for %s site", req.UserID, req.Locator.SiteID)
+			}
+			stats := userIDBkt.Stats()
+			count = stats.KeyN
+			return nil
+		})
+		return count, err
+	}
+
+	return 0, errors.Errorf("invalid count request %+v", req)
+}
+
+// Info get post(s) meta info
+func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return []store.PostInfo{}, err
+	}
+
+	if req.Locator.URL != "" { // post info
+		info := store.PostInfo{}
+		err = bdb.View(func(tx *bolt.Tx) error {
+			infoBkt := tx.Bucket([]byte(infoBucketName))
+			if e := b.load(infoBkt, req.Locator.URL, &info); e != nil {
+				return errors.Wrapf(e, "can't load info for %s", req.Locator.URL)
+			}
+			return nil
+		})
+
+		// set read-only from age and manual bucket
+		readOnlyAge := req.ReadOnlyAge
+		info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+		if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+			info.ReadOnly = true
+		}
+		return []store.PostInfo{info}, err
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		list := []store.PostInfo{}
+		err = bdb.View(func(tx *bolt.Tx) error {
+			postsBkt := tx.Bucket([]byte(postsBucketName))
+
+			c := postsBkt.Cursor()
+			n := 0
+			for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
+				n++
+				if req.Skip > 0 && n <= req.Skip {
+					continue
+				}
+				postURL := string(k)
+				infoBkt := tx.Bucket([]byte(infoBucketName))
+				info := store.PostInfo{}
+				if e := b.load(infoBkt, postURL, &info); e != nil {
+					return errors.Wrapf(e, "can't load info for %s", postURL)
+				}
+				list = append(list, info)
+				if req.Limit > 0 && len(list) >= req.Limit {
+					break
+				}
+			}
+			return nil
+		})
+		return list, err
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) {
+
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return nil, e
+	}
+
+	res = []interface{}{}
+	switch req.Flag {
+	case Verified:
+		err = bdb.View(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(verifiedBucketName))
+			_ = usersBkt.ForEach(func(k, _ []byte) error {
+				res = append(res, string(k))
+				return nil
+			})
+			return nil
+		})
+		return res, err
+	case Blocked:
+		err = bdb.View(func(tx *bolt.Tx) error {
+			bucket := tx.Bucket([]byte(blocksBucketName))
+			return bucket.ForEach(func(k []byte, v []byte) error {
+				ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+				if errParse != nil {
+					return errors.Wrap(errParse, "can't parse block ts")
+				}
+				if time.Now().Before(ts) {
+					// get user name from comment user section
+					userName := ""
+					findReq := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1}
+					userComments, errUser := b.Find(findReq)
+					if errUser == nil && len(userComments) > 0 {
+						userName = userComments[0].User.Name
+					}
+					res = append(res, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
+				}
+				return nil
+			})
+		})
+		return res, err
+	}
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// Delete post(s) by id or by userID
+func (b *BoltDB) Delete(req DeleteRequest) error {
+
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return e
+	}
+
+	switch {
+	case req.Locator.URL != "" && req.CommentID != "":
+		return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
+		return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode)
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
+		return b.deleteAll(bdb, req.Locator.SiteID)
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+// Close boltdb store
+func (b *BoltDB) Close() error {
+	errs := new(multierror.Error)
+	for site, db := range b.dbs {
+		err := errors.Wrapf(db.Close(), "can't close site %s", site)
+		errs = multierror.Append(errs, err)
+	}
+	return errs.ErrorOrNil()
+}
+
+// Last returns up to max last comments for given siteID
+func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+
+	comments = []store.Comment{}
+
+	if max > lastLimit || max == 0 {
+		max = lastLimit
+	}
+
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return nil, err
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		lastBkt := tx.Bucket([]byte(lastBucketName))
+		c := lastBkt.Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+
+			if !since.IsZero() {
+				// stop if reached "since" ts
+				tsSince := []byte(since.Format(tsNano))
+				if bytes.Compare(k, tsSince) <= 0 {
+					break
+				}
+			}
+			url, commentID, e := b.parseRef(v)
+			if e != nil {
+				return e
+			}
+			postBkt, e := b.getPostBucket(tx, url)
+			if e != nil {
+				return e
+			}
+
+			comment := store.Comment{}
+			if e = b.load(postBkt, commentID, &comment); e != nil {
+				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
+				continue
+			}
+			if comment.Deleted {
+				continue
+			}
+			comments = append(comments, comment)
+			if len(comments) >= max {
+				break
+			}
+		}
+		return nil
+	})
+
+	return comments, err
+}
+
+// userComments extracts all comments for given site and given userID
+// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
+func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+
+	comments = []store.Comment{}
+	commentRefs := []string{}
+
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return nil, err
+	}
+
+	if limit == 0 || limit > userLimit {
+		limit = userLimit
+	}
+
+	// get list of references to comments
+	err = bdb.View(func(tx *bolt.Tx) error {
+		usersBkt := tx.Bucket([]byte(userBucketName))
+		userIDBkt := usersBkt.Bucket([]byte(userID))
+		if userIDBkt == nil {
+			return errors.Errorf("no comments for user %s in store", userID)
+		}
+
+		c := userIDBkt.Cursor()
+		skipComments := 0
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			if len(commentRefs) >= limit {
+				break
+			}
+			if skip > 0 && skipComments < skip {
+				skipComments++
+				continue
+			}
+			commentRefs = append(commentRefs, string(v))
+		}
+		return nil
+	})
+
+	if err != nil {
+		return comments, err
+	}
+
+	// retrieve comments for refs
+	for _, v := range commentRefs {
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
+		}
+		getReq := GetRequest{Locator: store.Locator{SiteID: siteID, URL: url}, CommentID: commentID}
+		if c, errRef := b.Get(getReq); errRef == nil {
+			comments = append(comments, c)
+		}
+	}
+
+	return comments, err
+}
+
+func (b *BoltDB) checkFlag(req FlagRequest) (val bool) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return false
+	}
+
+	key := req.Locator.URL
+	if req.UserID != "" {
+		key = req.UserID
+	}
+
+	if req.Flag == Blocked {
+		var blocked bool
+		_ = bdb.View(func(tx *bolt.Tx) error {
+			bucket := tx.Bucket([]byte(blocksBucketName))
+			v := bucket.Get([]byte(key))
+			if v == nil {
+				blocked = false
+				return nil
+			}
+
+			until, e := time.Parse(tsNano, string(v))
+			if e != nil {
+				blocked = false
+				return nil
+			}
+			blocked = time.Now().Before(until)
+			return nil
+		})
+		return blocked
+	}
+
+	_ = bdb.View(func(tx *bolt.Tx) error {
+		var bucket *bolt.Bucket
+		if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+			return err
+		}
+		val = bucket.Get([]byte(key)) != nil
+		return nil
+	})
+	return val
+}
+
+func (b *BoltDB) setFlag(req FlagRequest) (res bool, err error) {
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return false, e
+	}
+
+	key := req.Locator.URL
+	if req.UserID != "" {
+		key = req.UserID
+	}
+
+	err = bdb.Update(func(tx *bolt.Tx) error {
+		var bucket *bolt.Bucket
+		if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+			return err
+		}
+		switch req.Update {
+		case FlagTrue:
+			if req.Flag == Blocked {
+				val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
+				if req.TTL > 0 {
+					val = time.Now().Add(req.TTL).Format(tsNano)
+				}
+				if e = bucket.Put([]byte(key), []byte(val)); e != nil {
+					return errors.Wrapf(e, "failed to put blocked to %s", key)
+				}
+				res = true
+				return nil
+			}
+
+			if e = bucket.Put([]byte(key), []byte(time.Now().Format(tsNano))); e != nil {
+				return errors.Wrapf(e, "failed to set flag %s for %s", req.Flag, req.Locator.URL)
+			}
+			res = true
+			return nil
+		case FlagFalse:
+			if e = bucket.Delete([]byte(key)); e != nil {
+				return errors.Wrapf(e, "failed to clean flag %s for %s", req.Flag, req.Locator.URL)
+			}
+			res = false
+		}
+		return nil
+	})
+
+	return res, err
+}
+
+func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error) {
+	switch flag {
+	case ReadOnly:
+		bkt = tx.Bucket([]byte(readonlyBucketName))
+	case Blocked:
+		bkt = tx.Bucket([]byte(blocksBucketName))
+	case Verified:
+		bkt = tx.Bucket([]byte(verifiedBucketName))
+	default:
+		return nil, errors.Errorf("unsupported flag %v", flag)
+	}
+	return bkt, nil
+}
+
+func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
+
+	return bdb.Update(func(tx *bolt.Tx) error {
+
+		postBkt, e := b.getPostBucket(tx, locator.URL)
+		if e != nil {
+			return e
+		}
+
+		comment := store.Comment{}
+		if e = b.load(postBkt, commentID, &comment); e != nil {
+			return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
+		}
+		// set deleted status and clear fields
+		comment.SetDeleted(mode)
+
+		if e = b.save(postBkt, commentID, comment); e != nil {
+			return errors.Wrapf(e, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
+		}
+
+		// delete from "last" bucket
+		lastBkt := tx.Bucket([]byte(lastBucketName))
+		if e = lastBkt.Delete([]byte(commentID)); e != nil {
+			return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
+		}
+
+		// decrement comments count for post url
+		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
+			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
+		}
+
+		return nil
+	})
+}
+
+// deleteAll removes all top-level buckets for given siteID
+func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
+
+	// delete all buckets except blocked users
+	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
+
+	// delete top-level buckets
+	err := bdb.Update(func(tx *bolt.Tx) error {
+		for _, bktName := range toDelete {
+
+			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
+				return errors.Wrapf(e, "failed to delete top level bucket %s", bktName)
+			}
+			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+				return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+			}
+		}
+		return nil
+	})
+
+	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
+}
+
+// deleteUser removes all comments for given user. Everything will be market as deleted
+// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
+func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
+
+	// get list of all comments outside of transaction loop
+	posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
+	if err != nil {
+		return err
+	}
+
+	type commentInfo struct {
+		locator   store.Locator
+		commentID string
+	}
+
+	// get list of commentID for all user's comment
+	comments := []commentInfo{}
+	for _, postInfo := range posts {
+		err = bdb.View(func(tx *bolt.Tx) error {
+			postsBkt := tx.Bucket([]byte(postsBucketName))
+			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
+			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+				comment := store.Comment{}
+				if err = json.Unmarshal(commentVal, &comment); err != nil {
+					return errors.Wrap(err, "failed to unmarshal")
+				}
+				if comment.User.ID == userID {
+					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
+				}
+				return nil
+			})
+			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Printf("[DEBUG] comments for removal=%d", len(comments))
+
+	// delete collected comments
+	for _, ci := range comments {
+		if e := b.deleteComment(bdb, ci.locator, ci.commentID, mode); e != nil {
+			return errors.Wrapf(err, "failed to delete comment %+v", ci)
+		}
+	}
+
+	//  delete  user bucket in hard mode
+	if mode == store.HardDelete {
+		err = bdb.Update(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(userBucketName))
+			if usersBkt != nil {
+				if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
+					return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
+				}
+			}
+			return nil
+		})
+
+		if err != nil {
+			return errors.Wrap(err, "can't delete user meta")
+		}
+	}
+
+	if len(comments) == 0 {
+		return errors.Errorf("unknown user %s", userID)
+	}
+
+	return err
+}
+
+// getPostBucket return bucket with all comments for postURL
+func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+	postsBkt := tx.Bucket([]byte(postsBucketName))
+	if postsBkt == nil {
+		return nil, errors.Errorf("no bucket %s", postsBucketName)
+	}
+	res := postsBkt.Bucket([]byte(postURL))
+	if res == nil {
+		return nil, errors.Errorf("no bucket %s in store", postURL)
+	}
+	return res, nil
+}
+
+// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
+func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+	postsBkt := tx.Bucket([]byte(postsBucketName))
+	if postsBkt == nil {
+		return nil, errors.Errorf("no bucket %s", postsBucketName)
+	}
+	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
+	if err != nil {
+		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
+	}
+	return res, nil
+}
+
+func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
+	usersBkt := tx.Bucket([]byte(userBucketName))
+	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
+	if e != nil {
+		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
+	}
+	return userIDBkt, nil
+}
+
+// save marshaled value to key for bucket. Should run in update tx
+func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
+	if value == nil {
+		return errors.Errorf("can't save nil value for %s", key)
+	}
+	jdata, jerr := json.Marshal(value)
+	if jerr != nil {
+		return errors.Wrap(jerr, "can't marshal comment")
+	}
+	if err = bkt.Put([]byte(key), jdata); err != nil {
+		return errors.Wrapf(err, "failed to save key %s", key)
+	}
+	return nil
+}
+
+// load and unmarshal json value by key from bucket. Should run in view tx
+func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
+	value := bkt.Get([]byte(key))
+	if value == nil {
+		return errors.Errorf("no value for %s", key)
+	}
+
+	if err := json.Unmarshal(value, &res); err != nil {
+		return errors.Wrap(err, "failed to unmarshal")
+	}
+	return nil
+}
+
+// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
+// it uses separate counts bucket because boltdb Stat call is very slow
+func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
+
+	infoBkt := tx.Bucket([]byte(infoBucketName))
+
+	info := store.PostInfo{}
+	if err := b.load(infoBkt, postURL, &info); err != nil {
+		info = store.PostInfo{}
+	}
+	if val == 0 { // get current count, don't update
+		return info.Count, nil
+	}
+	info.Count += val
+
+	return info.Count, b.save(infoBkt, postURL, &info)
+}
+
+func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
+	infoBkt := tx.Bucket([]byte(infoBucketName))
+	info := store.PostInfo{}
+	if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
+		info = store.PostInfo{
+			Count:   0,
+			URL:     comment.Locator.URL,
+			FirstTS: comment.Timestamp,
+			LastTS:  comment.Timestamp,
+		}
+	}
+	info.Count++
+	info.LastTS = comment.Timestamp
+	return info, b.save(infoBkt, comment.Locator.URL, &info)
+}
+
+func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
+	if res, ok := b.dbs[siteID]; ok {
+		return res, nil
+	}
+	return nil, errors.Errorf("site %q not found", siteID)
+}
+
+// makeRef creates reference combining url and comment id
+func (b *BoltDB) makeRef(comment store.Comment) []byte {
+	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
+}
+
+// parseRef gets parts of reference
+func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
+	elems := strings.Split(string(val), "!!")
+	if len(elems) != 2 {
+		return "", "", errors.Errorf("invalid reference value %s", string(val))
+	}
+	return elems[0], elems[1], nil
+}

backend/app/store/engine/bolt_accessor.go

@@ -1,537 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
-	dbs map[string]*bolt.DB
-}
-
-const (
-	// top level buckets
-	postsBucketName    = "posts"
-	lastBucketName     = "last"
-	userBucketName     = "users"
-	blocksBucketName   = "block"
-	infoBucketName     = "info"
-	readonlyBucketName = "readonly"
-	verifiedBucketName = "verified"
-
-	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-)
-
-// BoltSite defines single site param
-type BoltSite struct {
-	FileName string // full path to boltdb
-	SiteID   string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
-	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
-	result := BoltDB{dbs: make(map[string]*bolt.DB)}
-	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
-		}
-
-		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
-			infoBucketName, readonlyBucketName, verifiedBucketName}
-		err = db.Update(func(tx *bolt.Tx) error {
-			for _, bktName := range topBuckets {
-				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
-				}
-			}
-			return nil
-		})
-
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create top level bucket)")
-		}
-
-		result.dbs[site.SiteID] = db
-	}
-	return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
-	bdb, err := b.db(comment.Locator.SiteID)
-	if err != nil {
-		return "", err
-	}
-
-	if b.IsReadOnly(comment.Locator) {
-		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
-	}
-
-	err = bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
-		if e != nil {
-			return e
-		}
-
-		// check if key already in store, reject doubles
-		if postBkt.Get([]byte(comment.ID)) != nil {
-			return errors.Errorf("key %s already in store", comment.ID)
-		}
-
-		// serialize comment to json []byte for bolt and save
-		if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
-			return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
-		}
-
-		ref := b.makeRef(comment)
-
-		// add reference to comment to "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		e = lastBkt.Put(commentTs, ref)
-		if e != nil {
-			return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
-		}
-
-		// add reference to commentID to "users" bucket
-		userBkt, e := b.getUserBucket(tx, comment.User.ID)
-		if e != nil {
-			return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
-		}
-		// put into individual user's bucket with ts as a key
-		if e = userBkt.Put(commentTs, ref); e != nil {
-			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
-		}
-
-		// set info with the count for post url
-		if _, e = b.setInfo(tx, comment); e != nil {
-			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
-		}
-		return nil
-	})
-
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		return bucket.ForEach(func(k, v []byte) error {
-			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
-				return errors.Wrap(e, "failed to unmarshal")
-			}
-			comments = append(comments, comment)
-			return nil
-		})
-	})
-
-	comments = sortComments(comments, sortFld)
-	return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
-
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		c := lastBkt.Cursor()
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			url, commentID, e := b.parseRef(v)
-			if e != nil {
-				return e
-			}
-			postBkt, e := b.getPostBucket(tx, url)
-			if e != nil {
-				return e
-			}
-
-			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
-				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
-				continue
-			}
-			if comment.Deleted {
-				continue
-			}
-			comments = append(comments, comment)
-			if len(comments) >= max {
-				break
-			}
-		}
-		return nil
-	})
-
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return 0, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		var e error
-		count, e = b.count(tx, locator.URL, 0)
-		return e
-	})
-
-	return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		postsBkt := tx.Bucket([]byte(postsBucketName))
-
-		c := postsBkt.Cursor()
-		n := 0
-		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			n++
-			if skip > 0 && n <= skip {
-				continue
-			}
-			postURL := string(k)
-			infoBkt := tx.Bucket([]byte(infoBucketName))
-			info := store.PostInfo{}
-			if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
-				return errors.Wrapf(e, "can't load info for %s", postURL)
-			}
-			list = append(list, info)
-			if limit > 0 && len(list) >= limit {
-				break
-			}
-		}
-		return nil
-	})
-
-	return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return store.PostInfo{}, err
-	}
-
-	info := store.PostInfo{}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		infoBkt := tx.Bucket([]byte(infoBucketName))
-		if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
-			return errors.Wrapf(e, "can't load info for %s", locator.URL)
-		}
-		return nil
-	})
-
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if b.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-	commentRefs := []string{}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	if limit == 0 || limit > userLimit {
-		limit = userLimit
-	}
-
-	// get list of references to comments
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-
-		c := userIDBkt.Cursor()
-		skipComments := 0
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			if len(commentRefs) >= limit {
-				break
-			}
-			if skip > 0 && skipComments < skip {
-				skipComments++
-				continue
-			}
-			commentRefs = append(commentRefs, string(v))
-		}
-		return nil
-	})
-
-	if err != nil {
-		return comments, err
-	}
-
-	// retrieve comments for refs
-	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
-		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
-			comments = append(comments, c)
-		}
-	}
-
-	return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return 0, err
-	}
-	count := 0
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-		stats := userIDBkt.Stats()
-		count = stats.KeyN
-		return nil
-	})
-	return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return comment, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.load(bucket, []byte(commentID), &comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
-	if curComment, err := b.Get(locator, comment.ID); err == nil {
-		// preserve immutable fields
-		comment.ParentID = curComment.ParentID
-		comment.Locator = curComment.Locator
-		comment.Timestamp = curComment.Timestamp
-		comment.User = curComment.User
-	}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.save(bucket, []byte(comment.ID), comment)
-	})
-}
-
-// Close boltdb store
-func (b *BoltDB) Close() error {
-	errs := new(multierror.Error)
-	for site, db := range b.dbs {
-		err := errors.Wrapf(db.Close(), "can't close site %s", site)
-		errs = multierror.Append(errs, err)
-	}
-	return errs.ErrorOrNil()
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res := postsBkt.Bucket([]byte(postURL))
-	if res == nil {
-		return nil, errors.Errorf("no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
-	if err != nil {
-		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
-	usersBkt := tx.Bucket([]byte(userBucketName))
-	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
-	if e != nil {
-		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
-	}
-	return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
-	if value == nil {
-		return errors.Errorf("can't save nil value for %s", key)
-	}
-	jdata, jerr := json.Marshal(value)
-	if jerr != nil {
-		return errors.Wrap(jerr, "can't marshal comment")
-	}
-	if err = bkt.Put(key, jdata); err != nil {
-		return errors.Wrapf(err, "failed to save key %s", key)
-	}
-	return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
-	value := bkt.Get(key)
-	if value == nil {
-		return errors.Errorf("no value for %s", key)
-	}
-
-	if err := json.Unmarshal(value, &res); err != nil {
-		return errors.Wrap(err, "failed to unmarshal")
-	}
-	return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
-		info = store.PostInfo{}
-	}
-	if val == 0 { // get current count, don't update
-		return info.Count, nil
-	}
-	info.Count += val
-
-	return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
-		info = store.PostInfo{
-			Count:   0,
-			URL:     comment.Locator.URL,
-			FirstTS: comment.Timestamp,
-			LastTS:  comment.Timestamp,
-		}
-	}
-	info.Count++
-	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
-	if res, ok := b.dbs[siteID]; ok {
-		return res, nil
-	}
-	return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
-	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
-	elems := strings.Split(string(val), "!!")
-	if len(elems) != 2 {
-		return "", "", errors.Errorf("invalid reference value %s", string(val))
-	}
-	return elems[0], elems[1], nil
-}

backend/app/store/engine/bolt_accessor_test.go

@@ -1,374 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
-	assert.Equal(t, "key id-1 already in store", err.Error())
-
-	_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-
-	assert.NoError(t, b.Close())
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	comment := store.Comment{
-		ID:        "id-ro",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	err := b.SetReadOnly(comment.Locator, true)
-	require.Nil(t, err)
-
-	_, err = b.Create(comment)
-	assert.NotNil(t, err)
-	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
-	err = b.SetReadOnly(comment.Locator, false)
-	require.Nil(t, err)
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err)
-
-	_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = b.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = b.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = b.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	_, err = b.Last("bad", 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-
-	_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
-	res, err := b.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = b.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = b.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = b.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	_, err = b.List("bad", 1, 1)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	_, err = b.User("bad", "user1", 1, 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.User("radio-t", "userZ", 1, 0)
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
-	os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	defer os.Remove(testDb)
-
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err = b.Create(c)
-		require.Nil(t, err)
-	}
-
-	// get all comments
-	res, err := b.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = b.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	count, err := b.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	_, err = b.UserCount("bad", "user1")
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.UserCount("radio-t", "userZ")
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
-	b := BoltDB{}
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	res := b.makeRef(comment)
-	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
-	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/2", url)
-	assert.Equal(t, "12345", id)
-
-	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
-	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
-	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
-	os.Remove(testDb)
-
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b := boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

backend/app/store/engine/bolt_admin.go

@@ -1,339 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
-			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
-		}
-		// set deleted status and clear fields
-		comment.SetDeleted(mode)
-
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
-			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
-		}
-
-		// delete from "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
-			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
-		}
-
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
-		return nil
-	})
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
-	// delete top-level buckets
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		for _, bktName := range toDelete {
-
-			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
-			}
-			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-			}
-		}
-		return nil
-	})
-
-	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// get list of all comments outside of transaction loop
-	posts, err := b.List(siteID, 0, 0)
-	if err != nil {
-		return err
-	}
-
-	type commentInfo struct {
-		locator   store.Locator
-		commentID string
-	}
-
-	// get list of commentID for all user's comment
-	comments := []commentInfo{}
-	for _, postInfo := range posts {
-		err = bdb.View(func(tx *bolt.Tx) error {
-			postsBkt := tx.Bucket([]byte(postsBucketName))
-			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
-				comment := store.Comment{}
-				if err = json.Unmarshal(commentVal, &comment); err != nil {
-					return errors.Wrap(err, "failed to unmarshal")
-				}
-				if comment.User.ID == userID {
-					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
-				}
-				return nil
-			})
-			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
-	// delete collected comments
-	for _, ci := range comments {
-		if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
-			return errors.Wrapf(err, "failed to delete comment %+v", ci)
-		}
-	}
-
-	//  delete  user bucket
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		if usersBkt != nil {
-			if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
-				return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "can't delete user meta")
-	}
-
-	if len(comments) == 0 {
-		return errors.Errorf("unknown user %s", userID)
-	}
-
-	return err
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		switch status {
-		case true:
-			val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
-			if ttl > 0 {
-				val = time.Now().Add(ttl).Format(tsNano)
-			}
-			if e := bucket.Put([]byte(userID), []byte(val)); e != nil {
-				return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
-			}
-		}
-		return nil
-	})
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		val := bucket.Get([]byte(userID))
-		if val == nil {
-			blocked = false
-			return nil
-		}
-
-		until, err := time.Parse(tsNano, string(val))
-		if err != nil {
-			blocked = false
-			return nil
-		}
-		blocked = time.Now().Before(until)
-		return nil
-	})
-	return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
-			}
-			if time.Now().Before(ts) {
-				// get user name from comment user section
-				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
-					userName = userComments[0].User.Name
-				}
-				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
-			}
-			return nil
-		})
-	})
-
-	return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
-			}
-		case false:
-			if e := bucket.Delete([]byte(locator.URL)); e != nil {
-				return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
-			}
-		}
-		return nil
-	})
-}
-
-// IsReadOnly checks if post in RO mode
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		ro = bucket.Get([]byte(locator.URL)) != nil
-		return nil
-	})
-	return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set verified status for %s", userID)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean verified status for %s", userID)
-			}
-		}
-		return nil
-	})
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		verified = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return verified
-}
-
-// Verified returns list of verified userIDs
-func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(verifiedBucketName))
-		_ = usersBkt.ForEach(func(k, _ []byte) error {
-			ids = append(ids, string(k))
-			return nil
-		})
-		return nil
-	})
-	return ids, err
-}

backend/app/store/engine/bolt_admin_test.go

@@ -1,228 +0,0 @@
-package engine
-
-import (
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	err = b.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	err = b.DeleteAll("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	err := b.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	_, err = b.User("radio-t", "user1", 5, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	err = b.DeleteUser("radio-t-bad", "user1")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0))
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
-	assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0))
-
-	assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-	time.Sleep(50 * time.Millisecond)
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
-	assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0))
-
-	ids, err := b.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	time.Sleep(50 * time.Millisecond)
-	ids, err = b.Blocked("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-
-	_, err = b.Blocked("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, b.SetVerified("radio-t", "u1", false))
-	assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u2", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u3", false))
-
-	ids, err := b.Verified("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
-	_, err = b.Verified("radio-t-bad")
-	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
-}

backend/app/store/engine/bolt_test.go

@@ -0,0 +1,845 @@
+package engine
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestBoltDB_CreateAndFind(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	var bb Interface = b
+	_ = bb
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+	t.Log(res[0].ID)
+
+	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	assert.NotNil(t, err)
+	assert.Equal(t, "key id-1 already in store", err.Error())
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+	assert.NoError(t, b.Close())
+}
+
+func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.NotNil(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestBoltDB_Get(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.NotNil(t, err)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_Update(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	comment.Locator.SiteID = "radio-t"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
+}
+
+func TestBoltDB_FindLast(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_FindLastSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindInPostSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindForUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `no comments for user userZ in store`)
+}
+
+func TestBoltDB_FindForUserPagination(t *testing.T) {
+	_ = os.Remove(testDb)
+	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.Nil(t, err)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("id-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err = b.Create(c)
+		require.Nil(t, err)
+	}
+
+	// get all comments
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 5, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+	assert.Equal(t, "id-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, "id-189", res[0].ID)
+	assert.Equal(t, "id-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 5, len(res))
+	assert.Equal(t, "id-4", res[0].ID)
+	assert.Equal(t, "id-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_CountPost(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_CountUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`)
+}
+
+func TestBoltDB_InfoPost(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.NotNil(t, err)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.NotNil(t, err)
+
+	fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestBoltDB_InfoList(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.Nil(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
+
+	req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	_, err = b.Info(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_FlagBlockedUser(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err, "non-existing user can't be blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked on wrong site")
+}
+
+func TestBolt_FlagReadOnlyPost(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.Equal(t, true, val)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro on wrong site")
+}
+
+func TestBolt_FlagVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	isVerified := func(site, user string) bool {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`)
+	assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", FlagFalse))
+}
+
+func TestBolt_FlagListVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	ids, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	_, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
+}
+
+func TestBolt_FlagListBlocked(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error {
+		req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
+
+	vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	assert.Equal(t, 2, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+	assert.Equal(t, "user2", blockedList[1].ID)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	assert.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	_, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.EqualError(t, err, `site "bad" not found`)
+
+}
+
+func TestBolt_DeleteComment(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 2, count, "count=2 initially")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.NotNil(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
+}
+
+func TestBolt_DeleteHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestBolt_DeleteAll(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.EqualError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltDB_ref(t *testing.T) {
+	b := BoltDB{}
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	res := b.makeRef(comment)
+	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
+
+	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
+	assert.NoError(t, err)
+	assert.Equal(t, "https://radio-t.com/2", url)
+	assert.Equal(t, "12345", id)
+
+	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
+	assert.NotNil(t, err)
+}
+
+func TestBoltDB_NewFailed(t *testing.T) {
+	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
+	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
+	_ = os.Remove(testDb)
+
+	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	assert.Nil(t, err)
+	b = boltStore
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	teardown = func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
+	return b, teardown
+}
+
+func getReq(locator store.Locator, commentID string) GetRequest {
+	return GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/app/store/engine/engine.go

@@ -1,6 +1,7 @@
+package engine
+
 // Package engine defines interfaces each supported storage should implement.
 // Includes default implementation with boltdb
-package engine
 
 import (
 	"sort"
@@ -10,50 +11,82 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
-//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go"
+// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
+//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
 
-// Interface combines all store interfaces
+// Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Accessor
+	Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
-	Admin
+	Update(comment store.Comment) error                         // update comment, mutable parts only
+	Get(req GetRequest) (store.Comment, error)                  // get comment by id
+	Find(req FindRequest) ([]store.Comment, error)              // find comments for locator or site
+	Info(req InfoRequest) ([]store.PostInfo, error)             // get post(s) meta info
+	Count(req FindRequest) (int, error)                         // get count for post or user
+	Delete(req DeleteRequest) error                             // delete post(s) by id or by userID
+	Flag(req FlagRequest) (bool, error)                         // set and get flags
+	ListFlags(req FlagRequest) ([]interface{}, error)           // get list of flagged keys, like blocked & verified user
+	Close() error                                               // close storage engine
 }
 
-// UserRequest is the request send to get comments by user
+// GetRequest is the input for Get func
-type UserRequest struct {
+type GetRequest struct {
-	SiteID string
+	Locator   store.Locator `json:"locator"`
-	UserID string
+	CommentID string        `json:"comment_id"`
-	Limit  int
-	Skip   int
 }
 
-// Accessor defines all usual access ops avail for regular user
+// FindRequest is the input for all find operations
-type Accessor interface {
+type FindRequest struct {
-	Create(comment store.Comment) (commentID string, err error)           // create new comment, avoid dups by id
+	Locator store.Locator `json:"locator"`           // lack of URL means site operation
-	Get(locator store.Locator, commentID string) (store.Comment, error)   // get comment by id
+	UserID  string        `json:"user_id,omitempty"` // presence of UserID treated as user-related find
-	Put(locator store.Locator, comment store.Comment) error               // update comment, mutable parts only
+	Sort    string        `json:"sort,omitempty"`    // sort order with +/-field syntax
-	Find(locator store.Locator, sort string) ([]store.Comment, error)     // find comments for locator
+	Since   time.Time     `json:"since,omitempty"`   // time limit for found results
-	Last(siteID string, limit int) ([]store.Comment, error)               // last comments for given site, sorted by time
+	Limit   int           `json:"limit,omitempty"`
-	User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
+	Skip    int           `json:"skip,omitempty"`
-	UserCount(siteID, userID string) (int, error)                         // comments count by user
-	Count(locator store.Locator) (int, error)                             // number of comments for the post
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)    // list of commented posts
-	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)  // get post info
-	Close() error                                                         // close/stop engine
 }
 
-// Admin defines all store ops avail for admin only
+// InfoRequest is the input of Info operation used to get meta data about posts
-type Admin interface {
+type InfoRequest struct {
-	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
+	Locator     store.Locator `json:"locator"`
-	DeleteAll(siteID string) error                                               // delete all data from site
+	Limit       int           `json:"limit,omitempty"`
-	DeleteUser(siteID string, userID string) error                               // remove all comments from user
+	Skip        int           `json:"skip,omitempty"`
-	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent)
+	ReadOnlyAge int           `json:"ro_age,omitempty"`
-	IsBlocked(siteID string, userID string) bool                                 // check if user blocked
+}
-	Blocked(siteID string) ([]store.BlockedUser, error)                          // get list of blocked users
+
-	SetReadOnly(locator store.Locator, status bool) error                        // set/reset read-only flag
+// DeleteRequest is the input for all delete operations (comments, sites, users)
-	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
+type DeleteRequest struct {
-	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
+	Locator    store.Locator    `json:"locator"` // lack of URL means site operation
-	IsVerified(siteID string, userID string) bool                                // check verified status
+	CommentID  string           `json:"comment_id,omitempty"`
-	Verified(siteID string) ([]string, error)                                    // list of verified user ids
+	UserID     string           `json:"user_id,omitempty"`
+	DeleteMode store.DeleteMode `json:"del_mode"`
+}
+
+// Flag defines type of binary attribute
+type Flag string
+
+// FlagStatus represents values of the flag update
+type FlagStatus int
+
+// enum of update values
+const (
+	FlagNonSet FlagStatus = 0
+	FlagTrue   FlagStatus = 1
+	FlagFalse  FlagStatus = -1
+)
+
+// Enum of all flags
+const (
+	ReadOnly = Flag("readonly")
+	Verified = Flag("verified")
+	Blocked  = Flag("blocked")
+)
+
+// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
+type FlagRequest struct {
+	Flag    Flag          `json:"flag"`              // flag type
+	Locator store.Locator `json:"locator"`           // post locator
+	UserID  string        `json:"user_id,omitempty"` // for flags setting user status
+	Update  FlagStatus    `json:"update,omitempty"`  // if FlagNonSet it will be get op, if set will set the value
+	TTL     time.Duration `json:"ttl,omitempty"`     // ttl for time-sensitive flags only, like blocked for some period
 }
 
 const (
@@ -62,8 +95,8 @@ const (
 	userLimit = 500
 )
 
-// sortComments is for engines can't sort data internally
+// SortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
+func SortComments(comments []store.Comment, sortFld string) []store.Comment {
 	sort.Slice(comments, func(i, j int) bool {
 		switch sortFld {
 		case "+time", "-time", "time", "+active", "-active", "active":
@@ -84,6 +117,18 @@ func sortComments(comments []store.Comment, sortFld string) []store.Comment {
 			}
 			return comments[i].Score < comments[j].Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortFld, "-") {
+				if comments[i].Controversy == comments[j].Controversy {
+					return comments[i].Timestamp.Before(comments[j].Timestamp)
+				}
+				return comments[i].Controversy > comments[j].Controversy
+			}
+			if comments[i].Controversy == comments[j].Controversy {
+				return comments[i].Timestamp.Before(comments[j].Timestamp)
+			}
+			return comments[i].Controversy < comments[j].Controversy
+
 		default:
 			return comments[i].Timestamp.Before(comments[j].Timestamp)
 		}

backend/app/store/engine/engine_mock.go

@@ -0,0 +1,205 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package engine
+
+import mock "github.com/stretchr/testify/mock"
+import store "github.com/umputun/remark/backend/app/store"
+
+// MockInterface is an autogenerated mock type for the Interface type
+type MockInterface struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *MockInterface) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Count provides a mock function with given fields: req
+func (_m *MockInterface) Count(req FindRequest) (int, error) {
+	ret := _m.Called(req)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(FindRequest) int); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Create provides a mock function with given fields: comment
+func (_m *MockInterface) Create(comment store.Comment) (string, error) {
+	ret := _m.Called(comment)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(store.Comment) string); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(store.Comment) error); ok {
+		r1 = rf(comment)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Delete provides a mock function with given fields: req
+func (_m *MockInterface) Delete(req DeleteRequest) error {
+	ret := _m.Called(req)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Find provides a mock function with given fields: req
+func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.Comment
+	if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.Comment)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Flag provides a mock function with given fields: req
+func (_m *MockInterface) Flag(req FlagRequest) (bool, error) {
+	ret := _m.Called(req)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Get provides a mock function with given fields: req
+func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 store.Comment
+	if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(store.Comment)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(GetRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Info provides a mock function with given fields: req
+func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.PostInfo
+	if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.PostInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(InfoRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListFlags provides a mock function with given fields: req
+func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) {
+	ret := _m.Called(req)
+
+	var r0 []interface{}
+	if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]interface{})
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Update provides a mock function with given fields: comment
+func (_m *MockInterface) Update(comment store.Comment) error {
+	ret := _m.Called(comment)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(store.Comment) error); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

backend/app/store/engine/engine_test.go

@@ -11,33 +11,45 @@ import (
 
 func TestEngine_sortComments(t *testing.T) {
 	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
+		{ID: "1", Score: 5, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
+		{ID: "2", Score: 4, Controversy: 2, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
+		{ID: "3", Score: 6, Controversy: 3, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
+		{ID: "4", Score: 6, Controversy: 1, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
 	}
 
-	sortComments(cc, "+time")
+	SortComments(cc, "+time")
 	assert.Equal(t, "1", cc[0].ID)
 	assert.Equal(t, "2", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-time")
+	SortComments(cc, "-time")
 	assert.Equal(t, "4", cc[0].ID)
 	assert.Equal(t, "3", cc[1].ID)
 	assert.Equal(t, "2", cc[2].ID)
 	assert.Equal(t, "1", cc[3].ID)
 
-	sortComments(cc, "score")
+	SortComments(cc, "score")
 	assert.Equal(t, "2", cc[0].ID)
 	assert.Equal(t, "1", cc[1].ID)
 	assert.Equal(t, "3", cc[2].ID)
 	assert.Equal(t, "4", cc[3].ID)
 
-	sortComments(cc, "-score")
+	SortComments(cc, "-score")
 	assert.Equal(t, "3", cc[0].ID)
 	assert.Equal(t, "4", cc[1].ID)
 	assert.Equal(t, "1", cc[2].ID)
 	assert.Equal(t, "2", cc[3].ID)
+
+	SortComments(cc, "controversy")
+	assert.Equal(t, "1", cc[0].ID)
+	assert.Equal(t, "4", cc[1].ID)
+	assert.Equal(t, "2", cc[2].ID)
+	assert.Equal(t, "3", cc[3].ID)
+
+	SortComments(cc, "-controversy")
+	assert.Equal(t, "3", cc[0].ID)
+	assert.Equal(t, "2", cc[1].ID)
+	assert.Equal(t, "1", cc[2].ID)
+	assert.Equal(t, "4", cc[3].ID)
 }

backend/app/store/engine/mongo.go

@@ -1,377 +0,0 @@
-package engine
-
-import (
-	"time"
-
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	"github.com/go-pkgz/mongo"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Mongo implements engine interface
-type Mongo struct {
-	conn       *mongo.Connection
-	postWriter mongo.BufferedWriter
-}
-
-const (
-	mongoPosts     = "posts"
-	mongoMetaPosts = "meta_posts"
-	mongoMetaUsers = "meta_users"
-)
-
-type metaPost struct {
-	ID       string `bson:"_id"` // url
-	SiteID   string `bson:"site"`
-	ReadOnly bool   `bson:"read_only"`
-}
-
-type metaUser struct {
-	ID           string    `bson:"_id"` // user_id
-	SiteID       string    `bson:"site"`
-	Verified     bool      `bson:"verified"`
-	Blocked      bool      `bson:"blocked"`
-	BlockedUntil time.Time `bson:"blocked_until"`
-}
-
-// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
-// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
-func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
-	writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
-	result := Mongo{conn: conn, postWriter: writer}
-	err := result.prepare()
-	return &result, errors.Wrap(err, "failed to prepare mongo")
-}
-
-// Create new comment, write can be buffered and delayed.
-func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
-	// err = m.postWriter.Write(comment)
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Insert(&comment)
-	})
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).Sort(sortFld).All(&comments)
-	})
-	return comments, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).One(&comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
-			bson.M{"$set": bson.M{
-				"text":    comment.Text,
-				"orig":    comment.Orig,
-				"score":   comment.Score,
-				"votes":   comment.Votes,
-				"pin":     comment.Pin,
-				"deleted": comment.Deleted,
-			}})
-	})
-}
-
-// Last returns up to max last comments for given siteID
-func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "delete": false}
-		return coll.Find(query).Sort("-time").Limit(max).All(&comments)
-	})
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (m *Mongo) Count(locator store.Locator) (count int, err error) {
-
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
-		count, err = coll.Find(query).Count()
-		return err
-	})
-	return count, e
-}
-
-// List returns list of all commented posts with counters
-func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-	list = []store.PostInfo{}
-
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": siteID}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-			{"$skip": skip},
-			{"$limit": limit},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	return list, errors.Wrap(err, "can't get list")
-}
-
-// Info returns time range and count for locator
-func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
-	list := []store.PostInfo{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	if err != nil {
-		return info, err
-	}
-	if len(list) == 0 {
-		return info, errors.Errorf("can't load info for %s", locator.URL)
-	}
-	info = list[0]
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if m.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, nil
-}
-
-// User extracts all comments for given site and given userID
-func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "user.id": userID}
-		return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
-	})
-	return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
-}
-
-// UserCount returns number of comments for user
-func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		var e error
-		count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
-		return e
-	})
-	return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
-	return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
-		return e
-	})
-}
-
-// IsReadOnly checks if post in RO
-func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
-	meta := metaPost{}
-	err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
-	})
-	return err == nil && meta.ReadOnly
-}
-
-// SetVerified makes user verified or reset the flag
-func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
-		return e
-	})
-}
-
-// IsVerified checks if user verified
-func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Verified
-}
-
-// Verified returns list of verified user IDs
-func (m *Mongo) Verified(siteID string) (ids []string, err error) {
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
-	})
-	if err != nil {
-		return nil, err
-	}
-	for _, m := range metas {
-		ids = append(ids, m.ID)
-	}
-	return ids, nil
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	until := time.Time{}
-	if status {
-		until = time.Now().AddDate(100, 0, 0) // permanent is 50year
-		if ttl > 0 {
-			until = time.Now().Add(ttl)
-		}
-	}
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
-			bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
-		return errors.Wrapf(e, "failed to set block for %s", userID)
-	})
-}
-
-// IsBlocked checks if user blocked
-func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
-}
-
-// Blocked get lists of blocked users for given site
-func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID,
-			"blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
-	})
-	if err != nil {
-		return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
-	}
-
-	for _, mu := range metas {
-		blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
-		if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
-			blockedUser.Name = ucc[0].User.Name
-		}
-		users = append(users, blockedUser)
-	}
-	return users, nil
-}
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-	comment := store.Comment{}
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
-		if e != nil {
-			return e
-		}
-		comment.SetDeleted(mode)
-		return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
-	})
-	return errors.Wrapf(err, "can't delete %s", commentID)
-}
-
-// DeleteAll removes all info about siteID
-func (m *Mongo) DeleteAll(siteID string) error {
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		_, e := coll.RemoveAll(bson.M{"locator.site": siteID})
-		return e
-	})
-	return errors.Wrapf(err, "can't delete site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted".
-func (m *Mongo) DeleteUser(siteID string, userID string) error {
-	comments := []store.Comment{}
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
-		if e != nil {
-			return e
-		}
-		for _, c := range comments {
-			if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
-				return e
-			}
-		}
-		return nil
-	})
-}
-
-// Close boltdb store
-func (m *Mongo) Close() error {
-	if m.postWriter != nil {
-		return m.postWriter.Close()
-	}
-	return nil
-}
-
-// prepare collections with all indexes
-func (m *Mongo) prepare() error {
-	errs := new(multierror.Error)
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
-	})
-}
-
-func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-	return q.Skip(skip).Limit(limit)
-}

backend/app/store/engine/mongo_test.go

@@ -1,594 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/go-pkgz/mongo"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-func TestMongo_CreateAndFind(t *testing.T) {
-	var m Interface
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	require.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err, "reject dup")
-
-	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
-	assert.Nil(t, err)
-	assert.Equal(t, "id-3", id)
-	res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
-	assert.Nil(t, err)
-	require.Equal(t, 1, len(res))
-
-	assert.NoError(t, m.Close())
-}
-
-func TestMongo_Get(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err, "not found")
-}
-
-func TestMongo_Put(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = m.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = m.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `not found`)
-
-	err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `not found`)
-}
-
-func TestMongo_Last(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = m.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-}
-
-func TestMongo_Count(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-}
-
-func TestMongo_List(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := m.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
-
-	res, err := m.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = m.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = m.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = m.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = m.List("bad", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{}, res)
-}
-
-func TestMongo_Info(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := m.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-}
-
-func TestMongo_ReadOnly(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestMongo_Verified(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, m.SetVerified("radio-t", "u1", true))
-	assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, m.SetVerified("radio-t", "u1", false))
-	assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
-	assert.NoError(t, m.SetVerified("radio-t", "u1", true))
-	assert.NoError(t, m.SetVerified("radio-t", "u2", true))
-	assert.NoError(t, m.SetVerified("radio-t", "u3", false))
-
-	ids, err := m.Verified("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
-	ids, err = m.Verified("radio-t-bad")
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_GetForUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = m.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = m.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	res, err = m.User("bad", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_GetForUserPagination(t *testing.T) {
-	m, skip := prepMongo(t, false)
-	if skip {
-		return
-	}
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err := m.Create(c)
-		require.Nil(t, err, c.ID)
-	}
-
-	// get all comments
-	res, err := m.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = m.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = m.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = m.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_BlockUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
-	assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
-	assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
-
-	assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestMongo_BlockUserWithTTL(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
-	assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
-	time.Sleep(500 * time.Millisecond)
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestMongo_GetForUserCounter(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	count, err := m.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	count, err = m.UserCount("bad", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 0, count)
-}
-
-func TestMongo_BlockList(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
-	assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
-	assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
-
-	ids, err := m.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	time.Sleep(500 * time.Millisecond)
-	ids, err = m.Blocked("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-
-	ids, err = m.Blocked("bad")
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_Delete(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	err = m.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `can't delete id-1: not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `can't delete id-1: not found`)
-}
-
-func TestMongo_DeleteHard(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestMongo_DeleteAll(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-}
-
-func TestMongo_DeleteUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	err := m.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	cc, err := m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err, "no comments for user user1 in store")
-	assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-}
-
-func TestMongo_Parallel(t *testing.T) {
-	var m Interface
-	var skip bool
-	m, skip = prepMongoBuffered(t) // buffered engine, no comments
-	if skip {
-		return
-	}
-	go func() {
-		for i := 0; i < 100; i++ {
-			_, err := m.Create(store.Comment{
-				ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-			require.Nil(t, err)
-			time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
-		}
-	}()
-
-	for {
-		time.Sleep(10 * time.Millisecond)
-		res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-		assert.Nil(t, err)
-		if len(res) == 100 {
-			break
-		}
-	}
-}
-
-func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
-	conn, err := mongo.MakeTestConnection(t)
-	if err != nil {
-		return nil, true
-	}
-	mongo.RemoveTestCollection(t, conn)
-
-	m, err := NewMongo(conn, 1, 0*time.Microsecond)
-	require.Nil(t, err)
-
-	mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	if writeRecs {
-		_, err = m.Create(comment)
-		assert.Nil(t, err)
-	}
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	if writeRecs {
-		_, err = m.Create(comment)
-		assert.Nil(t, err)
-	}
-
-	return m, false
-}
-
-func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
-	conn, err := mongo.MakeTestConnection(t)
-	if err != nil {
-		return nil, true
-	}
-	mongo.RemoveTestCollection(t, conn)
-
-	m, err := NewMongo(conn, 10, 10*time.Millisecond)
-	mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
-
-	require.Nil(t, err)
-	return m, false
-}

backend/app/store/engine/remote.go

@@ -0,0 +1,104 @@
+package engine
+
+import (
+	"encoding/json"
+
+	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	rpc.Client
+}
+
+// Create comment and return ID
+func (r *RPC) Create(comment store.Comment) (commentID string, err error) {
+
+	resp, err := r.Call("store.create", comment)
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &commentID)
+	return commentID, err
+}
+
+// Get comment by ID
+func (r *RPC) Get(req GetRequest) (comment store.Comment, err error) {
+	resp, err := r.Call("store.get", req)
+	if err != nil {
+		return store.Comment{}, err
+	}
+
+	err = json.Unmarshal(*resp.Result, &comment)
+	return comment, err
+}
+
+// Update comment, mutable parts only
+func (r *RPC) Update(comment store.Comment) error {
+	_, err := r.Call("store.update", comment)
+	return err
+}
+
+// Find comments for locator
+func (r *RPC) Find(req FindRequest) (comments []store.Comment, err error) {
+	resp, err := r.Call("store.find", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &comments)
+	return comments, err
+}
+
+// Info returns post(s) meta info
+func (r *RPC) Info(req InfoRequest) (info []store.PostInfo, err error) {
+	resp, err := r.Call("store.info", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &info)
+	return info, err
+}
+
+// Flag sets and gets flags
+func (r *RPC) Flag(req FlagRequest) (status bool, err error) {
+	resp, err := r.Call("store.flag", req)
+	if err != nil {
+		return false, err
+	}
+	err = json.Unmarshal(*resp.Result, &status)
+	return status, err
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+func (r *RPC) ListFlags(req FlagRequest) (list []interface{}, err error) {
+	resp, err := r.Call("store.list_flags", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &list)
+	return list, err
+}
+
+// Count gets comments count by user or site
+func (r *RPC) Count(req FindRequest) (count int, err error) {
+	resp, err := r.Call("store.count", req)
+	if err != nil {
+		return 0, err
+	}
+	err = json.Unmarshal(*resp.Result, &count)
+	return count, err
+}
+
+// Delete post(s) by id or by userID
+func (r *RPC) Delete(req DeleteRequest) error {
+	_, err := r.Call("store.delete", req)
+	return err
+}
+
+// Close storage engine
+func (r *RPC) Close() error {
+	_, err := r.Call("store.close")
+	return err
+}

backend/app/store/engine/remote_test.go

@@ -0,0 +1,178 @@
+package engine
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestRemote_Create(t *testing.T) {
+	ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var eng Interface = &c
+	_ = eng
+
+	res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"},
+		Text: "msg"})
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Get(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	res, err := c.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_GetWithErrorResult(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "failed")
+}
+
+func TestRemote_GetWithErrorDecode(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "failed to decode response for store.get: EOF")
+}
+
+func TestRemote_GetWithErrorRemote(t *testing.T) {
+	c := RPC{Client: rpc.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.NotNil(t, err)
+	assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error())
+}
+
+func TestRemote_FailedStatus(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		t.Logf("req: %s", string(body))
+		w.WriteHeader(400)
+	}))
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "bad status 400 Bad Request for store.get")
+}
+
+func TestRemote_Update(t *testing.T) {
+	ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	err := c.Update(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"},
+		Text: "msg"})
+	assert.NoError(t, err)
+
+}
+
+func TestRemote_Find(t *testing.T) {
+	ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Find(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func TestRemote_Info(t *testing.T) {
+	ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Info(InfoRequest{Locator: store.Locator{URL: "http://example.com/url"},
+		Limit: 10, Skip: 5, ReadOnlyAge: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res)
+}
+
+func TestRemote_Flag(t *testing.T) {
+	ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Flag(FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: Verified})
+	assert.NoError(t, err)
+	assert.Equal(t, false, res)
+}
+
+func TestRemote_ListFlag(t *testing.T) {
+	ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked})
+	assert.NoError(t, err)
+	assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
+}
+
+func TestRemote_Count(t *testing.T) {
+	ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Count(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 11, res)
+}
+
+func TestRemote_Delete(t *testing.T) {
+	ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`,
+		`{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	err := c.Delete(DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+	assert.NoError(t, err)
+}
+
+func TestRemote_Close(t *testing.T) {
+	ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	err := c.Close()
+	assert.NoError(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/store/formatter.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
+	bf "gopkg.in/russross/blackfriday.v2"
 )
 
 // CommentFormatter implements all generic formatting ops on comment
@@ -40,10 +40,16 @@ func (f *CommentFormatter) Format(c Comment) Comment {
 
 // FormatText converts text with markdown processor, applies external converters and shortens links
 func (f *CommentFormatter) FormatText(txt string) (res string) {
-	mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
+	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
-		blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
+		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
-		blackfriday.BackslashLineBreak | blackfriday.Autolink
+		bf.BackslashLineBreak | bf.Autolink
-	res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt)))
+
+	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{
+		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
+	})
+
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
+	res = f.unEscape(res)
 
 	for _, conv := range f.converters {
 		res = conv.Convert(res)
@@ -85,3 +91,16 @@ func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTM
 	}
 	return resHTML
 }
+
+func (f *CommentFormatter) unEscape(txt string) (res string) {
+	elems := []struct {
+		from, to string
+	}{
+		{`—`, "—"},
+	}
+	res = txt
+	for _, e := range elems {
+		res = strings.Replace(res, e.from, e.to, -1)
+	}
+	return res
+}

backend/app/store/formatter_test.go

@@ -14,17 +14,23 @@ func (m mockConverter) Convert(text string) string { return text + "!converted"
 func TestFormatter_FormatText(t *testing.T) {
 	tbl := []struct {
 		in, out string
+		name    string
 	}{
-		{"", "!converted"},
+		{"", "!converted", "empty"},
-		{"12345 abc", "<p>12345 abc</p>\n!converted"},
+		{"12345 abc", "<p>12345 abc</p>\n!converted", "simple"},
-		{"**xyz** _aaa_", "<p><strong>xyz</strong> <em>aaa</em></p>\n!converted"},
+		{"**xyz** _aaa_ - \"sfs\"", "<p><strong>xyz</strong> <em>aaa</em> – «sfs»</p>\n!converted", "format"},
 		{
-			"http://127.0.0.1/some-long-link/12345/678901234567890", "<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0.1/some-long-link/12345/6789012...</a></p>\n!converted",
+			"http://127.0.0.1/some-long-link/12345/678901234567890",
+			"<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0." +
+				"1/some-long-link/12345/6789012...</a></p>\n!converted", "links",
 		},
+		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
 	}
 	f := NewCommentFormatter(mockConverter{})
-	for n, tt := range tbl {
+	for _, tt := range tbl {
-		assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n)
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.out, f.FormatText(tt.in))
+		})
 	}
 }
 

backend/app/store/image/fs_store.go

@@ -0,0 +1,177 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"io/ioutil"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+	MaxHeight  int
+	MaxWidth   int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	data, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't read source data for image %s", fileName)
+	}
+	if len(data) > f.MaxSize {
+		return "", errors.Errorf("file %s is too large (limit=%d)", fileName, f.MaxSize)
+	}
+
+	// read header first, needs it to check if data is valid png/gif/jpeg
+	if !isValidImage(data[:512]) {
+		return "", errors.Errorf("file %s is not in allowed format", fileName)
+	}
+
+	data, resized := resize(data, f.MaxWidth, f.MaxHeight)
+
+	id = path.Join(userID, guid()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+	if resized { // resized also converted to png
+		id = strings.TrimSuffix(id, filepath.Ext(id)) + ".png"
+		dst = f.location(f.Staging, id)
+	}
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	if err = ioutil.WriteFile(dst, data, 0600); err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+
+	log.Printf("[DEBUG] file %s saved for image %s, size=%d", dst, fileName, len(data))
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return nil
+	}
+
+	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", fpath, age)
+			rmErr := os.Remove(fpath)
+			_ = os.Remove(path.Dir(fpath)) // try to remove directory
+			return rmErr
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// SizeLimit returns max size of allowed image
+func (f *FileSystem) SizeLimit() int {
+	return f.MaxSize
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}

backend/app/store/image/fs_store_test.go

@@ -0,0 +1,312 @@
+package image
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveWithResize(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 32, 32
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1142, len(data))
+}
+
+func TestFsStore_SaveWithResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 400, 300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 10786, len(data))
+}
+
+func TestFsStore_SaveNoResizeJpeg(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxWidth, svc.MaxHeight = 1400, 1300
+	svc.MaxSize = 32000
+
+	fh, err := os.Open("testdata/circles.jpg")
+	defer func() { assert.NoError(t, fh.Close()) }()
+	assert.NoError(t, err)
+	id, err := svc.Save("circles.jpg", "user1", fh)
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".jpg")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 23983, len(data))
+}
+
+func TestFsStore_WrongFormat(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	_, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah bad image"))
+	assert.EqualError(t, err, "file file1.png is not in allowed format")
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", gopherPNG())
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 2000
+	_, err := svc.Save("blah_ff1.png", "user2", io.MultiReader(gopherPNG(), gopherPNG()))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", gopherPNG())
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(data))
+	assert.Equal(t, int64(1462), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string) (path string) {
+		id, err := svc.Save(file, user, gopherPNG())
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		assert.Equal(t, 1462, len(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(path.Dir(img1))
+	assert.NotNil(t, err, "no dir %s on staging anymore", path.Dir(img1))
+
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = &FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    1500,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}

backend/app/store/image/image.go

@@ -0,0 +1,201 @@
+// Package image handles storing, resizing and retrieval of images
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
+package image
+
+//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp image_mock.go"
+
+import (
+	"bytes"
+	"context"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	"image/png"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	"github.com/rs/xid"
+	"golang.org/x/image/draw"
+)
+
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
+	SizeLimit() int                                                          // max image size
+}
+
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	idsFn func() (ids []string)
+	TS    time.Time
+}
+
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil || s == nil {
+		return
+	}
+
+	s.once.Do(func() {
+		log.Printf("[DEBUG] image submitter activated")
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
+				}
+			}
+			log.Printf("[INFO] image submitter terminated")
+		}()
+	})
+
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
+}
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
+		}
+	}
+}
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the
+// biggest side (width or height) preserving aspect ratio.
+// Returns original data if resizing is not needed or failed.
+// If resized the result will be for png format and ok flag will be true.
+func resize(data []byte, limitW, limitH int) ([]byte, bool) {
+	if data == nil || limitW <= 0 || limitH <= 0 {
+		return data, false
+	}
+
+	src, _, err := image.Decode(bytes.NewBuffer(data))
+	if err != nil {
+		log.Printf("[WARN] can't decode image, %s", err)
+		return data, false
+	}
+
+	bounds := src.Bounds()
+	w, h := bounds.Dx(), bounds.Dy()
+	if w <= limitW && h <= limitH || w <= 0 || h <= 0 {
+		log.Printf("[DEBUG] resizing image is smaller that the limit or has 0 size")
+		return data, false
+	}
+
+	newW, newH := getProportionalSizes(w, h, limitW, limitH)
+	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
+	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+
+	var out bytes.Buffer
+	if err = png.Encode(&out, m); err != nil {
+		log.Printf("[WARN] can't encode resized image to png, %s", err)
+		return data, false
+	}
+	return out.Bytes(), true
+}
+
+// getProportionalSizes returns width and height resized by both dimensions proportionally
+func getProportionalSizes(srcW, srcH int, limitW, limitH int) (resW, resH int) {
+
+	if srcW <= limitW && srcH <= limitH {
+		return srcW, srcH
+	}
+
+	ratioW := float64(srcW) / float64(limitW)
+	propH := float64(srcH) / ratioW
+
+	ratioH := float64(srcH) / float64(limitH)
+	propW := float64(srcW) / ratioH
+
+	if int(propH) > limitH {
+		return int(propW), limitH
+	}
+
+	return limitW, int(propH)
+}
+
+// check if file f is a valid image format, i.e. gif, png, jpeg or webp
+func isValidImage(b []byte) bool {
+	ct := http.DetectContentType(b)
+	return ct == "image/gif" || ct == "image/png" || ct == "image/jpeg" || ct == "image/webp"
+}
+
+// guid makes a globally unique id
+func guid() string {
+	return xid.New().String()
+}

backend/app/store/image/image_mock.go

@@ -0,0 +1,105 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package image
+
+import context "context"
+import io "io"
+import mock "github.com/stretchr/testify/mock"
+import time "time"
+
+// MockStore is an autogenerated mock type for the Store type
+type MockStore struct {
+	mock.Mock
+}
+
+// Cleanup provides a mock function with given fields: ctx, ttl
+func (_m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
+	ret := _m.Called(ctx, ttl)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, time.Duration) error); ok {
+		r0 = rf(ctx, ttl)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Commit provides a mock function with given fields: id
+func (_m *MockStore) Commit(id string) error {
+	ret := _m.Called(id)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(id)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Load provides a mock function with given fields: id
+func (_m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	ret := _m.Called(id)
+
+	var r0 io.ReadCloser
+	if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
+		r0 = rf(id)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(io.ReadCloser)
+		}
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(string) int64); ok {
+		r1 = rf(id)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string) error); ok {
+		r2 = rf(id)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: fileName, userID, r
+func (_m *MockStore) Save(fileName string, userID string, r io.Reader) (string, error) {
+	ret := _m.Called(fileName, userID, r)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, io.Reader) string); ok {
+		r0 = rf(fileName, userID, r)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, io.Reader) error); ok {
+		r1 = rf(fileName, userID, r)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SizeLimit provides a mock function with given fields:
+func (_m *MockStore) SizeLimit() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}

backend/app/store/image/image_test.go

@@ -0,0 +1,156 @@
+package image
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"io/ioutil"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{ImageAPI: "/blah/"}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
+func TestService_ExtractPictures2(t *testing.T) {
+	svc := Service{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}
+	html := "<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." +
+		"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(ids), "one image in")
+	assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
+}
+
+func TestService_Cleanup(t *testing.T) {
+	store := MockStore{}
+	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)
+
+	svc := Service{Store: &store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+	store.AssertNumberOfCalls(t, "Cleanup", 10)
+}
+
+func TestService_Submit(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 0)
+	time.Sleep(time.Millisecond * 150)
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_Close(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	store := MockStore{}
+	store.On("Commit", mock.Anything, mock.Anything).Times(5).Return(nil)
+	svc := Service{Store: &store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
+	svc.Submit(nil)
+	store.AssertNumberOfCalls(t, "Commit", 3)
+	svc.Close()
+	store.AssertNumberOfCalls(t, "Commit", 5)
+}
+
+func TestService_resize(t *testing.T) {
+
+	// Reader is nil.
+	resized, ok := resize(nil, 100, 100)
+	assert.Nil(t, resized)
+	assert.False(t, ok)
+
+	// Negative limit error.
+	resized, ok = resize([]byte("some picture bin data"), -1, -1)
+	require.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("some picture bin data"))
+	assert.False(t, ok)
+
+	// Decode error.
+	resized, ok = resize([]byte("invalid image content"), 100, 100)
+	assert.NotNil(t, resized)
+	assert.Equal(t, resized, []byte("invalid image content"))
+	assert.False(t, ok)
+
+	cases := []struct {
+		file   string
+		wr, hr int
+	}{
+		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
+		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
+	}
+
+	for _, c := range cases {
+		img, err := ioutil.ReadFile(c.file)
+		require.Nil(t, err, "can't open test file %s", c.file)
+
+		// No need for resize, image dimensions are smaller than resize limit.
+		resized, ok = resize(img, 800, 800)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.Equal(t, resized, img)
+		assert.False(t, ok)
+
+		// Resizing to half of width. Check resized image format PNG.
+		resized, ok = resize(img, 400, 400)
+		assert.NotNil(t, resized, "file %s", c.file)
+		assert.True(t, ok)
+
+		imgRz, format, err := image.Decode(bytes.NewBuffer(resized))
+		assert.Nil(t, err, "file %s", c.file)
+		assert.Equal(t, "png", format, "file %s", c.file)
+		bounds := imgRz.Bounds()
+		assert.Equal(t, c.wr, bounds.Dx(), "file %s", c.file)
+		assert.Equal(t, c.hr, bounds.Dy(), "file %s", c.file)
+	}
+
+}
+
+func TestGetProportionalSizes(t *testing.T) {
+	tbl := []struct {
+		inpW, inpH     int
+		limitW, limitH int
+		resW, resH     int
+	}{
+		{10, 20, 50, 25, 10, 20},
+		{400, 200, 50, 25, 50, 25},
+		{100, 100, 50, 25, 25, 25},
+		{100, 200, 50, 25, 12, 25},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			resW, resH := getProportionalSizes(tt.inpW, tt.inpH, tt.limitW, tt.limitH)
+			assert.Equal(t, tt.resW, resW, "width")
+			assert.Equal(t, tt.resH, resH, "height")
+		})
+	}
+}

backend/app/store/service/restricted_words_test.go

@@ -1,8 +1,9 @@
 package service
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMatcher_Tokenize(t *testing.T) {

backend/app/store/service/service.go

@@ -1,7 +1,12 @@
+// Package service wraps engine interfaces with common logic unrelated to any particular engine implementation.
+// All consumers should be using service.DataStore and not the naked engine!
+
 package service
 
 import (
+	"math"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -14,17 +19,20 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // DataStore wraps store.Interface with additional methods
 type DataStore struct {
-	engine.Interface
+	Engine                 engine.Interface
 	EditDuration           time.Duration
 	AdminStore             admin.Store
 	MaxCommentSize         int
 	MaxVotes               int
+	PositiveScore          bool
 	TitleExtractor         *TitleExtractor
 	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -56,11 +64,13 @@ type PostMetaData struct {
 }
 
 const defaultCommentMaxSize = 2000
-const maxLastCommentsReply = 1000
+const maxLastCommentsReply = 5000
 
 // UnlimitedVotes doesn't restrict MaxVotes
 const UnlimitedVotes = -1
 
+var nonAdminUser = store.User{}
+
 // ErrRestrictedWordsFound returned in case comment text contains restricted words
 var ErrRestrictedWordsFound = errors.New("comment contains restricted words")
 
@@ -75,16 +85,92 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		return "", ErrRestrictedWordsFound
 	}
 
-	// keep input title and set to extracted if missing
+	func() { // keep input title and set to extracted if missing
-	if s.TitleExtractor != nil && comment.PostTitle == "" {
+		if s.TitleExtractor == nil || comment.PostTitle != "" {
-		if title, err := s.TitleExtractor.Get(comment.Locator.URL); err == nil {
+			return
-			comment.PostTitle = title
-		} else {
-			log.Printf("[WARN] failed to set title, %v", err)
 		}
+		title, e := s.TitleExtractor.Get(comment.Locator.URL)
+		if e != nil {
+			log.Printf("[WARN] failed to set title, %v", e)
+			return
+		}
+		comment.PostTitle = title
+	}()
+
+	s.submitImages(comment)
+	return s.Engine.Create(comment)
+}
+
+// Find wraps engine's Find call and alter results if needed. User used to alter comments
+// in order to differentiate between user's comments vs others comments.
+func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
+	return s.FindSince(locator, sort, user, time.Time{})
+}
+
+// FindSince wraps engine's Find call and alter results if needed. Returns comments after since tx
+func (s *DataStore) FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error) {
+	req := engine.FindRequest{Locator: locator, Sort: sort, Since: since}
+	comments, err := s.Engine.Find(req)
+	if err != nil {
+		return comments, err
 	}
 
-	return s.Interface.Create(comment)
+	changedSort := false
+	// set votes controversy for comments added prior to #274
+	for i, c := range comments {
+		if c.Controversy == 0 && len(c.Votes) > 0 {
+			c.Controversy = s.controversy(s.upsAndDowns(c))
+			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+				changedSort = true
+			}
+		}
+		comments[i] = s.alterComment(c, user)
+	}
+
+	// resort commits if altered
+	if changedSort {
+		comments = engine.SortComments(comments, sort)
+	}
+
+	return comments, nil
+}
+
+// Get comment by ID
+func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) {
+	c, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
+	if err != nil {
+		return store.Comment{}, err
+	}
+	return s.alterComment(c, user), nil
+}
+
+// Put updates comment, mutable parts only
+func (s *DataStore) Put(locator store.Locator, comment store.Comment) error {
+	comment.Locator = locator
+	return s.Engine.Update(comment)
+}
+
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
+	s.ImageService.Submit(func() []string {
+		c := comment
+		// this can be called after last edit, we have to retrieve fresh comment
+		cc, err := s.Engine.Get(engine.GetRequest{Locator: c.Locator, CommentID: c.ID})
+		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
+			return nil
+		}
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
 }
 
 // prepareNewComment sets new comment fields, hashing and sanitizing data
@@ -110,24 +196,31 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err
 	return comment, nil
 }
 
+// DeleteAll removes all data from site
+func (s *DataStore) DeleteAll(siteID string) error {
+	req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}}
+	return s.Engine.Delete(req)
+}
+
 // SetPin pin/un-pin comment as special
 func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
-	comment, err := s.Get(locator, commentID)
+	comment, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return err
 	}
 	comment.Pin = status
-	return s.Put(locator, comment)
+	comment.Locator = locator
+	return s.Engine.Update(comment)
 }
 
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -154,6 +247,10 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		return comment, errors.Errorf("maximum number of votes exceeded for comment %s", commentID)
 	}
 
+	if s.PositiveScore && comment.Score <= 0 && !val {
+		return comment, errors.Errorf("minimal score reached for comment %s", commentID)
+	}
+
 	// reset vote if user changed to opposite
 	if voted && v != val {
 		delete(comment.Votes, userID)
@@ -171,7 +268,34 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 		comment.Score--
 	}
 
-	return comment, s.Put(locator, comment)
+	comment.Vote = 0
+	if vv, ok := comment.Votes[userID]; ok {
+		if vv {
+			comment.Vote = 1
+		} else {
+			comment.Vote = -1
+		}
+	}
+
+	comment.Controversy = s.controversy(s.upsAndDowns(comment))
+	comment.Locator = locator
+	return comment, s.Engine.Update(comment)
+}
+
+// controversy calculates controversial index of votes
+// source - https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L60
+func (s *DataStore) controversy(ups, downs int) float64 {
+
+	if downs <= 0 || ups <= 0 {
+		return 0
+	}
+
+	magnitude := ups + downs
+	balance := float64(downs) / float64(ups)
+	if ups <= downs {
+		balance = float64(ups) / float64(downs)
+	}
+	return math.Pow(float64(magnitude), balance)
 }
 
 // EditRequest contains fields needed for comment update
@@ -184,7 +308,7 @@ type EditRequest struct {
 
 // EditComment to edit text and update Edit info
 func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -200,7 +324,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 
 	if req.Delete { // delete request
 		comment.Deleted = true
-		return comment, s.Delete(locator, commentID, store.SoftDelete)
+		delReq := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete}
+		return comment, s.Engine.Delete(delReq)
 	}
 
 	if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) {
@@ -213,9 +338,10 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 		Timestamp: time.Now(),
 		Summary:   req.Summary,
 	}
-
+	comment.Locator = locator
 	comment.Sanitize()
-	err = s.Put(locator, comment)
+
+	err = s.Engine.Update(comment)
 	return comment, err
 }
 
@@ -233,7 +359,8 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 		return true
 	}
 
-	comments, err := s.Last(comment.Locator.SiteID, maxLastCommentsReply)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply}
+	comments, err := s.Engine.Find(req)
 	if err != nil {
 		log.Printf("[WARN] can't get last comments for reply check, %v", err)
 		return false
@@ -250,13 +377,49 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 	return false
 }
 
+// UserReplies returns list of all comments replied to given user
+func (s *DataStore) UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error) {
+
+	comments, e := s.Last(siteID, maxLastCommentsReply, time.Time{}, nonAdminUser)
+	if e != nil {
+		return nil, "", errors.Wrap(e, "can't get last comments")
+	}
+	replies := []store.Comment{}
+
+	// get a comment for given userID in order to retrieve name
+	userName := ""
+	if cc, err := s.User(siteID, userID, 1, 0, nonAdminUser); err == nil && len(cc) > 0 {
+		userName = cc[0].User.Name
+	}
+
+	// collect replies
+	for _, c := range comments {
+
+		if len(replies) > limit || time.Since(c.Timestamp) > duration {
+			break
+		}
+
+		if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
+			var pc store.Comment
+			if pc, e = s.Get(c.Locator, c.ParentID, nonAdminUser); e != nil {
+				return nil, "", errors.Wrap(e, "can't get parent comment")
+			}
+			if pc.User.ID == userID {
+				replies = append(replies, c)
+			}
+		}
+	}
+
+	return replies, userName, nil
+}
+
 // SetTitle puts title from the locator.URL page and overwrites any existing title
 func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error) {
 	if s.TitleExtractor == nil {
 		return comment, errors.New("no title extractor")
 	}
 
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -267,7 +430,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
 		return comment, err
 	}
 	comment.PostTitle = title
-	err = s.Put(locator, comment)
+	comment.Locator = locator
+	err = s.Engine.Update(comment)
 	return comment, err
 }
 
@@ -275,7 +439,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
 func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
 	res := []store.PostInfo{}
 	for _, p := range postIDs {
-		if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
+		req := engine.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}}
+		if c, err := s.Engine.Count(req); err == nil {
 			res = append(res, store.PostInfo{URL: p, Count: c})
 		}
 	}
@@ -302,7 +467,11 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
 
 // IsAdmin checks if usesID in the list of admins
 func (s *DataStore) IsAdmin(siteID string, userID string) bool {
-	for _, a := range s.AdminStore.Admins(siteID) {
+	admins, err := s.AdminStore.Admins(siteID)
+	if err != nil {
+		return false
+	}
+	for _, a := range admins {
 		if a == userID {
 			return true
 		}
@@ -310,20 +479,126 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
 	return false
 }
 
+// IsReadOnly checks if post read-only
+func (s *DataStore) IsReadOnly(locator store.Locator) bool {
+	req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetReadOnly set/reset read-only flag
+func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+
+	}
+	req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly, Update: roStatus}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// IsVerified checks if user verified
+func (s *DataStore) IsVerified(siteID string, userID string) bool {
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetVerified set/reset verified status for user
+func (s *DataStore) SetVerified(siteID string, userID string, status bool) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+	}
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified, Update: roStatus}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// IsBlocked checks if user blocked
+func (s *DataStore) IsBlocked(siteID string, userID string) bool {
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Blocked}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetBlock set/reset verified status for user
+func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+	}
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID,
+		Flag: engine.Blocked, Update: roStatus, TTL: ttl}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// Blocked returns list with all blocked users
+func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) {
+	blocked, e := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Blocked})
+	if e != nil {
+		return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID)
+	}
+	for _, v := range blocked {
+		res = append(res, v.(store.BlockedUser))
+	}
+	return res, nil
+}
+
+// Info get post info
+func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
+	req := engine.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge}
+	res, err := s.Engine.Info(req)
+	if err != nil {
+		return store.PostInfo{}, err
+	}
+	if len(res) == 0 {
+		return store.PostInfo{}, errors.Errorf("post %+v not found", locator)
+	}
+	return res[0], nil
+}
+
+// Delete comment by id
+func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
+	req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
+	return s.Engine.Delete(req)
+}
+
+// DeleteUser removes all comments from user
+func (s *DataStore) DeleteUser(siteID string, userID string, mode store.DeleteMode) error {
+	req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: mode}
+	return s.Engine.Delete(req)
+}
+
+// List of commented posts
+func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip}
+	return s.Engine.Info(req)
+}
+
+// Count gets number of comments for the post
+func (s *DataStore) Count(locator store.Locator) (int, error) {
+	req := engine.FindRequest{Locator: locator}
+	return s.Engine.Count(req)
+}
+
 // Metas returns metadata for users and posts
 func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
 	umetas = []UserMetaData{}
 	pmetas = []PostMetaData{}
+
 	// set posts meta
-	posts, err := s.List(siteID, 0, 0)
+	posts, err := s.Engine.Info(engine.InfoRequest{Locator: store.Locator{SiteID: siteID}})
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID)
 	}
+
 	for _, p := range posts {
 		if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) {
 			pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true})
 		}
-
 	}
 
 	// set users meta
@@ -345,11 +620,12 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
 	}
 
 	// process verified users
-	verified, err := s.Verified(siteID)
+	verified, err := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Verified})
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID)
 	}
-	for _, v := range verified {
+	for _, vi := range verified {
+		v := vi.(string)
 		val, ok := m[v]
 		if !ok {
 			val = UserMetaData{ID: v}
@@ -390,8 +666,50 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
 	return errs.ErrorOrNil()
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
+// User gets comment for given userID on siteID
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) {
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip}
+	comments, err := s.Engine.Find(req)
+	if err != nil {
+		return comments, err
+	}
+	return s.alterComments(comments, user), nil
+}
+
+// UserCount is comments count by user
+func (s *DataStore) UserCount(siteID, userID string) (int, error) {
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID}
+	return s.Engine.Count(req)
+}
+
+// Last gets last comments for site, cross-post. Limited by count and optional since ts
+func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) {
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"}
+	comments, err := s.Engine.Find(req)
+	if err != nil {
+		return comments, err
+	}
+	return s.alterComments(comments, user), nil
+}
+
+// Close store service
+func (s *DataStore) Close() error {
+	return s.Engine.Close()
+}
+
+func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
+	for _, v := range c.Votes {
+		if v {
+			ups++
+			continue
+		}
+		downs++
+	}
+	return ups, downs
+}
+
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()
@@ -404,3 +722,53 @@ func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
 
 	return lock
 }
+
+func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []store.Comment) {
+	res = make([]store.Comment, len(cc))
+	for i, c := range cc {
+		res[i] = s.alterComment(c, user)
+	}
+	return res
+}
+
+func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) {
+
+	blocReq := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+	blocked, bErr := s.Engine.Flag(blocReq)
+
+	// mark user blocked
+	if bErr == nil && blocked {
+		c.User.Blocked = blocked
+	}
+
+	// set verified status retroactively
+	if !c.User.Blocked {
+		verifReq := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+		c.User.Verified, _ = s.Engine.Flag(verifReq)
+	}
+
+	// hide info from non-admins
+	if !user.Admin {
+		c.User.IP = ""
+	}
+
+	c = s.prepVotes(c, user)
+	return c
+}
+
+// prepare vote info for client view
+func (s *DataStore) prepVotes(c store.Comment, user store.User) store.Comment {
+
+	c.Vote = 0 // default is "none" (not voted)
+
+	if v, ok := c.Votes[user.ID]; ok {
+		if v {
+			c.Vote = 1
+		} else {
+			c.Vote = -1
+		}
+	}
+
+	c.Votes = nil // hide voters list
+	return c
+}

backend/app/store/service/title.go

@@ -3,19 +3,24 @@ package service
 import (
 	"io"
 	"net/http"
+	"strings"
+	"time"
 
+	"github.com/go-pkgz/lcw"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/pkg/errors"
 	"golang.org/x/net/html"
 )
 
-const teMaxCachedRecs = 1000
+const (
+	teCacheMaxRecs = 1000
+	teCacheTTL     = 15 * time.Minute
+)
 
 // TitleExtractor gets html title from remote page, cached
 type TitleExtractor struct {
 	client http.Client
-	cache  cache.LoadingCache
+	cache  lcw.LoadingCache
 }
 
 // NewTitleExtractor makes extractor with cache. If memory cache failed, switching to no-cache
@@ -24,23 +29,27 @@ func NewTitleExtractor(client http.Client) *TitleExtractor {
 		client: client,
 	}
 	var err error
-	res.cache, err = cache.NewMemoryCache(cache.MaxKeys(teMaxCachedRecs))
+	res.cache, err = lcw.NewExpirableCache(lcw.TTL(teCacheTTL), lcw.MaxKeySize(teCacheMaxRecs))
 	if err != nil {
-		log.Printf("[WARN] failed to make cache, %v", err)
+		log.Printf("[WARN] failed to make cache, caching disabled for titles, %v", err)
-		res.cache = &cache.Nop{}
+		res.cache = &lcw.Nop{}
 	}
 	return &res
 }
 
 // Get page for url and return title
 func (t *TitleExtractor) Get(url string) (string, error) {
-
+	client := http.Client{Timeout: t.client.Timeout, Transport: t.client.Transport}
-	b, err := t.cache.Get(cache.NewKey("site").ID(url), func() ([]byte, error) {
+	b, err := t.cache.Get(url, func() (lcw.Value, error) {
-		resp, err := t.client.Get(url)
+		resp, err := client.Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
-		defer resp.Body.Close() //nolint
+		defer func() {
+			if err = resp.Body.Close(); err != nil {
+				log.Printf("[WARN] failed to close title extractor body, %v", err)
+			}
+		}()
 		if resp.StatusCode != 200 {
 			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
 		}
@@ -49,14 +58,16 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if !ok {
 			return nil, errors.Errorf("can't get title for %s", url)
 		}
-		return []byte(title), nil
+		return title, nil
 	})
 
+	// on error save result (empty string) to cache too and return "" title
 	if err != nil {
+		_, _ = t.cache.Get(url, func() (lcw.Value, error) { return "", nil })
 		return "", err
 	}
 
-	return string(b), nil
+	return b.(string), nil
 }
 
 // get title from body reader, traverse recursively
@@ -75,7 +86,10 @@ func (t *TitleExtractor) isTitleElement(n *html.Node) bool {
 
 func (t *TitleExtractor) traverse(n *html.Node) (string, bool) {
 	if t.isTitleElement(n) {
-		return n.FirstChild.Data, true
+		title := n.FirstChild.Data
+		title = strings.Replace(title, "\n", "", -1)
+		title = strings.TrimSpace(title)
+		return title, true
 	}
 
 	for c := n.FirstChild; c != nil; c = c.NextSibling {

backend/app/store/service/title_test.go

@@ -1,14 +1,17 @@
 package service
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/syncs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -21,7 +24,8 @@ func TestTitle_GetTitle(t *testing.T) {
 		title string
 	}{
 		{`<html><title>blah 123</title><body> 2222</body></html>`, true, "blah 123"},
-		{`<html><title>blah 123 `, true, "blah 123 "},
+		{`<html><title>blah 123 `, true, "blah 123"},
+		{"<html><title>\n\n  blah 123 \n ", true, "blah 123"},
 		{`<html><body> 2222</body></html>`, false, ""},
 	}
 
@@ -41,7 +45,8 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>\n\n blah 123\n</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -55,9 +60,59 @@ func TestTitle_Get(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/good")
+		r, e := ex.Get(ts.URL + "/good")
-		require.Nil(t, err)
+		require.Nil(t, e)
-		assert.Equal(t, "blah 123", title)
+		assert.Equal(t, "blah 123", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
+
+func TestTitle_GetConcurrent(t *testing.T) {
+	body := ""
+	for n := 0; n < 1000; n++ {
+		body += "something something blah blah\n"
+	}
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.String(), "/good") {
+			atomic.AddInt32(&hits, 1)
+			_, err := w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			assert.NoError(t, err)
+			return
+		}
+		w.WriteHeader(404)
+	}))
+
+	g := syncs.NewSizedGroup(10)
+
+	for i := 0; i < 100; i++ {
+		ii := i
+		g.Go(func(_ context.Context) {
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(ii))
+			require.Nil(t, err)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(ii), title)
+		})
+	}
+	g.Wait()
+	assert.Equal(t, int32(100), atomic.LoadInt32(&hits))
+}
+
+func TestTitle_GetFailed(t *testing.T) {
+	ex := NewTitleExtractor(http.Client{Timeout: 5 * time.Second})
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(404)
+	}))
+
+	_, err := ex.Get(ts.URL + "/bad")
+	require.NotNil(t, err)
+
+	for i := 0; i < 100; i++ {
+		r, e := ex.Get(ts.URL + "/bad")
+		require.Nil(t, e)
+		assert.Equal(t, "", r)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
+}

backend/app/store/service/tree.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"sort"
@@ -111,8 +111,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 }
 
 // filter returns comments for parentID
-func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) (f []store.Comment) {
+func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) []store.Comment {
-
+	f := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			f = append(f, c)
@@ -151,6 +151,18 @@ func (t *Tree) sortNodes(sortType string) {
 			}
 			return t.Nodes[i].Comment.Score < t.Nodes[j].Comment.Score
 
+		case "+controversy", "-controversy", "controversy":
+			if strings.HasPrefix(sortType, "-") {
+				if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+					return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+				}
+				return t.Nodes[i].Comment.Controversy > t.Nodes[j].Comment.Controversy
+			}
+			if t.Nodes[i].Comment.Controversy == t.Nodes[j].Comment.Controversy {
+				return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
+			}
+			return t.Nodes[i].Comment.Controversy < t.Nodes[j].Comment.Controversy
+
 		default:
 			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
 		}

backend/app/store/service/tree_test.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"encoding/json"
@@ -96,15 +96,15 @@ func TestTreeSortNodes(t *testing.T) {
 	comments := []store.Comment{
 		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
+		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2, Controversy: 10},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
+		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3, Controversy: 5},
 		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
 		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
 		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
 		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
 		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
 		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
+		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2, Controversy: 7},
 		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
 		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
 		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
@@ -140,6 +140,19 @@ func TestTreeSortNodes(t *testing.T) {
 	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
 	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
 
+	res = MakeTree(comments, "+controversy", 0)
+	assert.Equal(t, "3", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[3].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[4].Comment.ID)
+
+	res = MakeTree(comments, "-controversy", 0)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "4", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "2", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[3].Comment.ID)
+
 	res = MakeTree(comments, "undefined", 0)
 	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
 	assert.Equal(t, "1", res.Nodes[0].Comment.ID)

backend/app/store/user.go

@@ -2,7 +2,7 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1"
+	"crypto/sha1" // nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -41,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id)
+	return hashWithFallback(sha1.New(), id) // nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -0,0 +1,41 @@
+module github.com/umputun/remark/backend
+
+go 1.12
+
+replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1
+
+require (
+	cloud.google.com/go v0.41.0 // indirect
+	github.com/PuerkitoBio/goquery v1.5.0
+	github.com/coreos/bbolt v1.3.3
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/didip/tollbooth v4.0.0+incompatible
+	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
+	github.com/go-chi/chi v4.0.2+incompatible
+	github.com/go-chi/cors v1.0.0
+	github.com/go-chi/render v1.0.1
+	github.com/go-pkgz/auth v0.7.2
+	github.com/go-pkgz/lcw v0.3.1
+	github.com/go-pkgz/lgr v0.6.3
+	github.com/go-pkgz/repeater v1.1.2
+	github.com/go-pkgz/rest v1.4.1
+	github.com/go-pkgz/syncs v1.1.1
+	github.com/google/uuid v1.1.1
+	github.com/gorilla/feeds v1.1.1
+	github.com/hashicorp/go-multierror v1.0.0
+	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
+	github.com/kyokomi/emoji v2.1.0+incompatible
+	github.com/microcosm-cc/bluemonday v1.0.2
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/pkg/errors v0.8.1
+	github.com/rakyll/statik v0.1.6
+	github.com/rs/xid v1.2.1
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
+	golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
+	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	gopkg.in/russross/blackfriday.v2 v2.0.1
+)

backend/go.sum

@@ -0,0 +1,220 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM=
+cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
+github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.7.2 h1:+LvAgqwQtYuWphpZE8qLtspVd65+VgreJkFLsKrtmmk=
+github.com/go-pkgz/auth v0.7.2/go.mod h1:ibOpZYISiaOvAHe2bsKj2s3v4AkMam2WxxIFn+zhulo=
+github.com/go-pkgz/lcw v0.3.1 h1:PhfB0xNUawLMlx5rXvOTIc7d5LMrr1GM9vIzmG96aUI=
+github.com/go-pkgz/lcw v0.3.1/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
+github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
+github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
+github.com/go-pkgz/repeater v1.1.2 h1:OxTyUMdEGiN4jRk5g3HHWQ6o4GDezjCLwpmDa/On+mU=
+github.com/go-pkgz/repeater v1.1.2/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.1 h1:jWN+y6FS/Xe+8z4l3QMbSnODGyaxDHGojIS+wyKIjxg=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji v2.1.0+incompatible h1:+DYU2RgpI6OHG4oQkM5KlqD3Wd3UPEsX8jamTo1Mp6o=
+github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
+github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff h1:+2zgJKVDVAz/BWSsuniCmU1kLCjL88Z8/kv39xCI9NQ=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9 h1:uc17S921SPw5F2gJo7slQ3aqvr2RwpL7eb3+DZncu3s=
+golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/remark.rest

@@ -1,9 +1,13 @@
 
 ### find request with tree
-GET {{host}}/api/v1/find?site={{site}}&sort=-active&format=tree&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=tree&url={{url}}
 
 ### find request with plain
-GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=plain&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+
+### find request with plain. dev token for secret=secret, not admin
+GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### last 50 comments
 GET {{host}}/api/v1/last/50?site={{site}}
@@ -41,7 +45,7 @@ Content-Type: application/json
 }
 
 ### update comment
-PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site=remark&url=https://radio-t.com/blah1
+PUT {{host}}/api/v1/comment/7b88d7a91353ab206cb63cdca18fb26bcb30205b?site={{site}}&url=https://radio-t.com/blah1
 Content-Type: application/json
 
 {
@@ -50,22 +54,23 @@ Content-Type: application/json
 }
 
 ### pin comment
-PUT {{host}}/api/v1/admin/pin/3665976683?site=remark&url={{url}}&pin=1
+PUT {{host}}/api/v1/admin/pin/3665976683?site={{site}}&url={{url}}&pin=1
 
 ### vote for comment
-PUT {{host}}/api/v1/vote/3665976683?site=remark&url={{url}}&vote=1
+PUT http://127.0.0.1:8080/api/v1/vote/8a8c0b80-0d0a-41c3-84ad-f4034704e827?site={{site}}&url={{url}}&vote=-1
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
 ### get user info
 GET {{host}}/api/v1/user
 
 ### get comment by id
-GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
+GET {{host}}/api/v1/id/73e346f4-d57d-41a8-8803-6671aa187d8e?site={{site}}&url={{url}}
 
 ### get comment by id 2
-GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
+GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site={{site}}&url=https://radio-t.com/blah1
 
 ### get comment by user id
-GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
+GET {{host}}/api/v1/comments?site={{site}}&user={{user}}&limit=5
 
 ### get comment by user id2
 GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
@@ -84,7 +89,7 @@ Content-Type: application/json
 ]
 
 ### list commented posts
-GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
+GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=0
 
 ### block user
 PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1
@@ -99,7 +104,7 @@ GET {{host}}/api/v1/admin/blocked?site={{site}}
 DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
 
 ### get post info
-GET {{host}}/api/v1/info?site={{site}}&url=https://radio-t.com/p/2018/05/08/prep-597/
+GET {{host}}/api/v1/info?site={{site}}&url={{url}
 
 ### post rss
 GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
@@ -107,7 +112,7 @@ GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
 ### site rss
 GET {{host}}/api/v1/rss/site?site={{site}}
 
-### replie rss
+### reply rss
 GET {{host}}/api/v1/rss/reply?site={{site}}&user={{user}}
 
 ### get default avatar
@@ -116,5 +121,14 @@ GET {{host}}/api/v1/avatar/blah
 ### get config
 GET  {{host}}/api/v1/config?site={{site}}
 
+### deleteme (user's request). dev token for secret=secret, not admin
+POST {{host}}/api/v1/deleteme?site_id={{site}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
+
+### deletme, admin. admin token for secret=secret
+GET {{host}}/api/v1/admin/deleteme?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1NjU2NjI4MDYsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTU3NzEzOTQ2LCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiIiwiYXR0cnMiOnsiYWRtaW4iOmZhbHNlLCJibG9ja2VkIjpmYWxzZSwiZGVsZXRlX21lIjp0cnVlfX19.qmnsQt_jilHzoauA9D7t1m3w69qvAJsZkuKVPWhpdik
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE2NTc3MDQ5NzksImp0aSI6Ijk4ZDNhOGFkMGNjZWU5M2Q1MWYwYjJiOTY1ZjU2YmE2NmJkNmZiNzYiLCJpYXQiOjE1NTc3MDQ2NzksImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.YIt9Zq3n0O8PMkR78pGAqyHI0exCU0vLKjcHgcXfLFw
+
+
 ### ping
 GET {{host}}/ping

backend/vendor/cloud.google.com/go/AUTHORS

@@ -1,15 +0,0 @@
-# This is the official list of cloud authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as:
-# Name or Organization <email address>
-# The email address is not required for organizations.
-
-Filippo Valsorda <hi@filippo.io>
-Google Inc.
-Ingo Oeser <nightlyone@googlemail.com>
-Palm Stone Games, Inc.
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -1,39 +0,0 @@
-# People who have agreed to one of the CLAs and can contribute patches.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# https://developers.google.com/open-source/cla/individual
-# https://developers.google.com/open-source/cla/corporate
-#
-# Names should be added to this file as:
-#     Name <email address>
-
-# Keep the list alphabetically sorted.
-
-Alexis Hunt <lexer@google.com>
-Andreas Litt <andreas.litt@gmail.com>
-Andrew Gerrand <adg@golang.org>
-Brad Fitzpatrick <bradfitz@golang.org>
-Burcu Dogan <jbd@google.com>
-Dave Day <djd@golang.org>
-David Sansome <me@davidsansome.com>
-David Symonds <dsymonds@golang.org>
-Filippo Valsorda <hi@filippo.io>
-Glenn Lewis <gmlewis@google.com>
-Ingo Oeser <nightlyone@googlemail.com>
-Johan Euphrosine <proppy@google.com>
-Jonathan Amsterdam <jba@google.com>
-Kunpei Sakai <namusyaka@gmail.com>
-Luna Duclos <luna.duclos@palmstonegames.com>
-Magnus Hiie <magnus.hiie@gmail.com>
-Mario Castro <mariocaster@gmail.com>
-Michael McGreevy <mcgreevy@golang.org>
-Omar Jarjur <ojarjur@google.com>
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Sarah Adams <shadams@google.com>
-Thanatat Tamtan <acoshift@gmail.com>
-Toby Burress <kurin@google.com>
-Tuo Shan <shantuo@google.com>
-Tyler Treat <ttreat31@gmail.com>