add focus style to sort select

* break auth panel render into submethods

* move var definition

* break renderUnathorized into submethods

* hide login providers behind dropdown if they are exceed length of 3

* amend dropdown to behave nicely being placed in another dropdown

* add style to providers enclosed in dropdown

* add provider reducer and actions

* add provider save/restore to app flow

* place last login provider first in providers list

* infer StoreState from combineReducers return type

* move collapsed threads retoration to action

* fix: provider lost in other

* add dynamic threshold depending on window width

* fix & add tests

.dockerignore

@@ -20,4 +20,5 @@ debug.test
 *.prof
 *.test
 remark42
-/backend/var/
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -7,10 +7,6 @@ pipeline:
   build:
     image: golang:1.12-alpine
     commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
       - cd backend/app
       - go build -v ./...
 
@@ -94,8 +90,3 @@ pipeline:
     secrets: [ email_username, email_password ]
     when:
       status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]

.gitignore

@@ -19,3 +19,4 @@ debug.test
 remark42
 /bin/
 /backend/var/
+compose-private-backend.yml

.travis.yml

@@ -3,9 +3,6 @@ install:
   - docker-compose --version
 
 script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
   - docker build
       --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
       --build-arg CI=$CI
@@ -19,7 +16,4 @@ script:
       --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
       --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
       --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
       .
-  - docker rm -f mongo
-  

Dockerfile

@@ -19,26 +19,23 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
 
 ADD backend /build/backend
 ADD .git /build/.git
 WORKDIR /build/backend
 
+ENV GOFLAGS="-mod=vendor"
+
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
         golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
         --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
         --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
@@ -55,7 +52,7 @@ RUN \
     if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
-    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 
 FROM node:10.11-alpine as build-frontend-deps

Dockerfile.artifacts

@@ -64,32 +64,33 @@ RUN \
     echo "runs outside of drone" && tag=""; \
     else tag=_${DRONE_TAG}; fi && \
     apk add --no-cache --update zip && \
-    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
-    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
-    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
-    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
-    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
 
 # upload to github
-RUN \
+#RUN \
-    if [ -z "$DRONE_TAG" ] ; then \
+#    if [ -z "$DRONE_TAG" ] ; then \
-    echo "skip upload to github" ; \
+#    echo "skip upload to github" ; \
-    else \
+#    else \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
-    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
-    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+#    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
-    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
 
 FROM alpine
 COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/

README.md

@@ -3,9 +3,10 @@
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
 * Social login via Google, Facebook, Github and Yandex
+* Login via email 
 * Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus and wordpress
+* Import from Disqus and WordPress
 * Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
@@ -23,6 +24,16 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
+
+<details><summary>Screenshots</summary>
+
+Comments example:
+![](https://github.com/umputun/remark/blob/master/screenshots/comments.png)
+
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark/wiki/Admin-UI)
+</details>
+
 
 #
 
@@ -113,7 +124,7 @@ _this is the recommended way to run remark42_
 | image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
 | image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
 | image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
-| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                          |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `800`                    | width of resized image                           |
 | image.resize-height     | IMAGE_RESIZE_HEIGHT     | `300`                    | height of resized image                          |
 | auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
 | auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
@@ -127,6 +138,16 @@ _this is the recommended way to run remark42_
 | auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
 | auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
 | auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                            |
+| auth.email.host         | AUTH_EMAIL_HOST         |                          | smtp host                                        |
+| auth.email.port         | AUTH_EMAIL_PORT         | `25`                     | smtp port                                        |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                       |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                    |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                               |
+| auth.email.tls          | AUTH_EMAIL_TLS          | `false`                  | enable TLS                                       |
+| auth.email.user         | AUTH_EMAIL_USER         |                          | smtp user name                                   |
+| auth.email.passwd       | AUTH_EMAIL_PASSWD       |                          | smtp password                                    |
+| auth.email.timeout      | AUTH_EMAIL_TIMEOUT      | `10s`                    | smtp timeout                                     |
 | notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
 | notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
 | notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
@@ -148,6 +169,9 @@ _this is the recommended way to run remark42_
 | edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
 | read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
 | img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                             |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                  |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                        |
 | update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
 | admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
 | dbg                     | DEBUG                   | `false`                  | debug mode                                       |
@@ -308,18 +332,28 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['embed'] // optional param; which components to load. default to ["embed"]
+                          // to load all components define components as ['embed', 'last-comments', 'counter']
+                          // available component are:
+                          //     - 'embed': basic comments widget
+                          //     - 'last-comments': last comments widget, see `Last Comments` section below
+                          //     - 'counter': counter widget, see `Counter` section below
     url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -348,19 +382,24 @@ window.REMARK42.changeTheme('light');
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -376,19 +415,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
+  (function(c) {
-    var d = document, s = d.createElement('script');
+    for(var i = 0; i < c.length; i++){
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
+      var d = document, s = d.createElement('script');
-    (d.head || d.body).appendChild(s);
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
-  })();
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -561,7 +605,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
    }{}
 ```
 
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
 * `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
 * `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
   ```go
@@ -590,21 +634,58 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 
   ```go
   type Config struct {
-      Version        string   `json:"version"`
+     	Version        string   `json:"version"`
-      EditDuration   int      `json:"edit_duration"`
+        EditDuration   int      `json:"edit_duration"`
-      MaxCommentSize int      `json:"max_comment_size"`
+        MaxCommentSize int      `json:"max_comment_size"`
-      Admins         []string `json:"admins"`
+        Admins         []string `json:"admins"`
-      AdminEmail     string   `json:"admin_email"`
+        AdminEmail     string   `json:"admin_email"`
-      Auth           []string `json:"auth_providers"`
+        Auth           []string `json:"auth_providers"`
-      LowScore       int      `json:"low_score"`
+        LowScore       int      `json:"low_score"`
-      CriticalScore  int      `json:"critical_score"`
+        CriticalScore  int      `json:"critical_score"`
-      PositiveScore  bool     `json:"positive_score"`
+        PositiveScore  bool     `json:"positive_score"`
-      ReadOnlyAge    int      `json:"readonly_age"`
+        ReadOnlyAge    int      `json:"readonly_age"`
-      MaxImageSize   int      `json:"max_image_size"`
+        MaxImageSize   int      `json:"max_image_size"`
+        EmojiEnabled   bool     `json:"emoji_enabled"`
   }
   ```
 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
+* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
+
+### Streaming API
+
+Streaming API provide server-sent events for post updates as well as site update
+
+* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
+* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
+
+<details><summary>response example</summary>
+
+```
+data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}         
+```
+
+</details>
 
 ### RSS feeds
 
@@ -634,7 +715,7 @@ _returned id should be appended to load image url on caller side_
 * `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
 * `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
 * `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
-* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
+* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completion.
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
 * `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
 * `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.

backend/app/cmd/avatar.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"path"
-	"time"
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -78,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
 			return nil, err
@@ -93,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
 }
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}

backend/app/cmd/avatar_test.go

@@ -13,32 +13,15 @@ import (
 
 func TestAvatar_Execute(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
@@ -46,9 +29,9 @@ func TestAvatar_Execute(t *testing.T) {
 	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
+		"--dst.bolt.file=/tmp/ava-test2.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }

backend/app/cmd/cleanup.go

@@ -131,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL) //nolint
+	r, err := http.Get(listURL) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL) //nolint
+		r, err = http.Get(commentsURL) // nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}

backend/app/cmd/server.go

@@ -15,20 +15,22 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
+	"github.com/kyokomi/emoji"
 	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/provider"
+	"github.com/go-pkgz/auth/provider/sender"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/mongo"
 	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/migrator"
 	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest/api"
 	"github.com/umputun/remark/backend/app/rest/proxy"
+	"github.com/umputun/remark/backend/app/rpc"
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
@@ -41,11 +43,11 @@ type ServerCommand struct {
 	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
 	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
 	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
 	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+	Stream StreamGroup `group:"stream" namespace:"stream" env-namespace:"STREAM"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
 	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
@@ -63,6 +65,7 @@ type ServerCommand struct {
 	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
 	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
 	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	EnableEmoji     bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
 
 	Auth struct {
 		TTL struct {
@@ -75,6 +78,18 @@ type ServerCommand struct {
 		Yandex    AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
 		Dev       bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
 		Anonymous bool      `long:"anon" env:"ANON" description:"enable anonymous login"`
+		Email     struct {
+			Enable       bool          `long:"enable" env:"ENABLE" description:"enable auth via email"`
+			Host         string        `long:"host" env:"HOST" description:"smtp host"`
+			Port         int           `long:"port" env:"PORT" description:"smtp port"`
+			From         string        `long:"from" env:"FROM" description:"email's from"`
+			Subject      string        `long:"subj" env:"SUBJ" default:"remark42 confirmation" description:"email's subject"`
+			ContentType  string        `long:"content-type" env:"CONTENT_TYPE" default:"text/html" description:"content type"`
+			TLS          bool          `long:"tls" env:"TLS" description:"enable TLS"`
+			SMTPUserName string        `long:"user" env:"USER" description:"smtp user name"`
+			SMTPPassword string        `long:"passwd" env:"PASSWD" description:"smtp password"`
+			TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"10s" description:"smtp timeout"`
+		} `group:"email" namespace:"email" env-namespace:"EMAIL"`
 	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
 
 	CommonOpts
@@ -88,16 +103,17 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"rpc" default:"bolt"`
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"TPC"`
 }
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -113,7 +129,7 @@ type ImageGroup struct {
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
@@ -125,7 +141,7 @@ type AvatarGroup struct {
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
@@ -133,19 +149,14 @@ type CacheGroup struct {
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
-	URL string `long:"url" env:"URL" description:"mongo url"`
-	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"rpc" default:"shared"`
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+	RPC RPCGroup `group:"rpc" namespace:"rpc" env-namespace:"RPC"`
 }
 
 // NotifyGroup defines options for notification
@@ -170,6 +181,21 @@ type SSLGroup struct {
 	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
 }
 
+// StreamGroup define options for streaming apis
+type StreamGroup struct {
+	RefreshInterval time.Duration `long:"refresh" env:"REFRESH" default:"5s" description:"refresh interval for streams"`
+	TimeOut         time.Duration `long:"timeout" env:"TIMEOUT" default:"15m" description:"timeout to close streams on inactivity"`
+	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
+}
+
+// RPCGroup defines options for remote modules (plugins)
+type RPCGroup struct {
+	API          string        `long:"api" env:"API" description:"rpc extension api url"`
+	TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+	AuthUser     string        `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -238,9 +264,10 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make pictures store")
 	}
+	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
 
 	dataService := &service.DataStore{
-		Interface:              storeEngine,
+		Engine:                 storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
@@ -280,7 +307,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	}
 
 	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
-	commentFormatter := store.NewCommentFormatter(imgProxy)
+	emojiFmt := store.CommentConverterFunc(func(text string) string { return text })
+	if s.EnableEmoji {
+		emojiFmt = func(text string) string { return emoji.Sprint(text) }
+	}
+	commentFormatter := store.NewCommentFormatter(imgProxy, emojiFmt)
 
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
@@ -303,6 +334,12 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
 		ImageService:     imageService,
+		Streamer: &api.Streamer{
+			TimeOut:   s.Stream.TimeOut,
+			Refresh:   s.Stream.RefreshInterval,
+			MaxActive: int32(s.Stream.MaxActive),
+		},
+		EmojiEnabled: s.EnableEmoji,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -400,13 +437,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
-	case "mongo":
+	case "rpc":
-		mgServer, e := s.makeMongo()
+		r := &engine.RPC{Client: rpc.Client{
-		if e != nil {
+			API:        s.Store.RPC.API,
-			return result, errors.Wrap(e, "failed to create mongo server")
+			Client:     http.Client{Timeout: s.Store.RPC.TimeOut},
-		}
+			AuthUser:   s.Store.RPC.AuthUser,
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
+			AuthPasswd: s.Store.RPC.AuthPassword,
-		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
 	}
@@ -422,13 +460,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
-	case "mongo":
-		mgServer, err := s.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
 			return nil, err
@@ -453,7 +484,8 @@ func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 				MaxHeight:  s.Image.ResizeHeight,
 				MaxWidth:   s.Image.ResizeWidth,
 			},
-			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
+			ImageAPI: s.RemarkURL + "/api/v1/picture/",
+			TTL:      5 * s.EditDuration, // add extra time to image TTL for staging
 		}, nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
@@ -470,13 +502,14 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 			}
 		}
 		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
-	case "mongo":
+	case "rpc":
-		mgServer, e := s.makeMongo()
+		r := &admin.RPC{Client: rpc.Client{
-		if e != nil {
+			API:        s.Admin.RPC.API,
-			return nil, errors.Wrap(e, "failed to create mongo server")
+			Client:     http.Client{Timeout: s.Admin.RPC.TimeOut},
-		}
+			AuthUser:   s.Admin.RPC.AuthUser,
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
+			AuthPasswd: s.Admin.RPC.AuthPassword,
-		return admin.NewMongoStore(conn, s.SharedSecret), nil
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
 	}
@@ -488,26 +521,33 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
 	case "mem":
 		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
-	// case "mongo":
-	// 	mgServer, err := s.makeMongo()
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "failed to create mongo server")
-	// 	}
-	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
-	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-	// 		cache.MaxKeys(s.Cache.Max.Items))
 	case "none":
 		return &cache.Nop{}, nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
+var msgTemplate = `
-	if s.Mongo.URL == "" {
+<!DOCTYPE html>
-		return nil, errors.New("no mongo URL provided")
+<html>
-	}
+<head>
-	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
+	<meta name="viewport" content="width=device-width" />
-}
+	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+</head>
+<body>
+<div style="text-align: center; font-family: Arial, sans-serif; font-size: 18px;">
+	<h1 style="position: relative; color: #4fbbd6; margin-top: 0.2em;">Remark42</h1>
+	<p style="position: relative; max-width: 20em; margin: 0 auto 1em auto; line-height: 1.4em;">Confirmation&nbsp;for <b>{{.User}}</b> on&nbsp;site&nbsp;<b>{{.Site}}</b></p>
+	<div style="background-color: #eee; max-width: 20em; margin: 0 auto; border-radius: 0.4em; padding: 0.5em;">
+		<p style="position: relative; margin: 0 0 0.5em 0;">TOKEN</p>
+		<p style="position: relative; font-size: 0.7em; opacity: 0.8;"><i>Copy and&nbsp;paste this text into “token” field on&nbsp;comments page</i></p>
+		<p style="position: relative; font-family: monospace; background-color: #fff; margin: 0; padding: 0.5em; word-break: break-all; text-align: left; border-radius: 0.2em; -webkit-user-select: all; user-select: all;">{{.Token}}</p>
+	</div>
+	<p style="position: relative; margin-top: 2em; font-size: 0.8em; opacity: 0.8;"><i>Sent to {{.Address}}</i></p>
+</div>
+</body>
+</html>
+`
 
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
@@ -534,6 +574,22 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 
+	if s.Auth.Email.Enable {
+		params := sender.EmailParams{
+			Host:         s.Auth.Email.Host,
+			Port:         s.Auth.Email.Port,
+			From:         s.Auth.Email.From,
+			Subject:      s.Auth.Email.Subject,
+			ContentType:  s.Auth.Email.ContentType,
+			TLS:          s.Auth.Email.TLS,
+			SMTPUserName: s.Auth.Email.SMTPUserName,
+			SMTPPassword: s.Auth.Email.SMTPPassword,
+			TimeOut:      s.Auth.Email.TimeOut,
+		}
+		sndr := sender.NewEmailClient(params, log.Default())
+		authenticator.AddVerifProvider("email", msgTemplate, sndr)
+	}
+
 	if s.Auth.Anonymous {
 		log.Print("[INFO] anonymous access enabled")
 		var isValidAnonName = regexp.MustCompile(`^[a-zA-Z][\w ]+$`).MatchString
@@ -628,11 +684,13 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",
 		Logger:            log.Default(),
 		RefreshCache:      newAuthRefreshCache(),
+		UseGravatar:       true,
 	})
 	s.addAuthProviders(authenticator)
 	return authenticator

backend/app/cmd/server_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
@@ -12,28 +13,27 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
-	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
+	"github.com/jessevdk/go-flags"
-	flags "github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+		o.Port = port
 		return o
 	})
 
 	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
+	time.Sleep(250 * time.Millisecond) // let server start
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -43,24 +43,27 @@ func TestServerApp(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
-	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 	body, _ = ioutil.ReadAll(resp.Body)
 	t.Log(string(body))
 
-	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+	email, err := app.dataService.AdminStore.Email("")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@demo.remark42.com", email, "default admin email")
 
 	app.Wait()
 }
 
 func TestServerApp_DevMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
@@ -72,7 +75,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -84,8 +87,9 @@ func TestServerApp_DevMode(t *testing.T) {
 }
 
 func TestServerApp_AnonMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.Auth.Anonymous = true
 		return o
 	})
@@ -97,7 +101,7 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
 
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -106,80 +110,25 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// try to login with bad name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 403, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 403, resp.StatusCode)
 
 	app.Wait()
 }
-func TestServerApp_WithMongo(t *testing.T) {
-
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
-
-	opts := ServerCommand{}
-	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
-
-	// prepare options
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
-		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
-	require.Nil(t, err)
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
-
-	// create app
-	app, err := opts.newServerApp()
-	require.Nil(t, err)
-
-	defer func() {
-		s, e := mongo.NewServerWithURL(mongoURL, 10*time.Second)
-		assert.NoError(t, e)
-		conn := mongo.NewConnection(s, "test_remark", "")
-		_ = conn.WithDB(func(dbase *mgo.Database) error {
-			assert.NoError(t, dbase.DropDatabase())
-			return nil
-		})
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(5 * time.Second)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	go func() { _ = app.run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:12345/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
-
-	app.Wait()
-}
 
 func TestServerApp_WithSSL(t *testing.T) {
 	opts := ServerCommand{}
@@ -237,6 +186,45 @@ func TestServerApp_WithSSL(t *testing.T) {
 	app.Wait()
 }
 
+func TestServerApp_WithRemote(t *testing.T) {
+
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none",
+		"--store.type=rpc", "--store.rpc.api=http://127.0.0.1",
+		"--port=12345", "--admin.type=rpc", "--admin.rpc.api=http://127.0.0.1", "--avatar.fs.path=/tmp"})
+	require.Nil(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation, opts.Image.FS.Path = "/tmp", "/tmp"
+
+	// create app
+	app, err := opts.newServerApp()
+	require.Nil(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(5 * time.Second)
+		log.Print("[TEST] terminate app")
+		cancel()
+	}()
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	// send ping
+	resp, err := http.Get("http://localhost:12345/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	app.Wait()
+}
+
 func TestServerApp_Failed(t *testing.T) {
 	opts := ServerCommand{}
 	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
@@ -284,7 +272,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 func TestServerApp_Shutdown(t *testing.T) {
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+		o.Port = rand.Intn(40000) + 10000
 		return o
 	})
 	st := time.Now()
@@ -360,8 +348,9 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+		o.Port = port
 		return o
 	})
 
@@ -390,17 +379,18 @@ func TestServerAuthHooks(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
+	req, e := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
 	assert.Nil(t, e)
 	req.SetBasicAuth("admin", "password")
 	resp, e = client.Do(req)
@@ -414,10 +404,10 @@ func TestServerAuthHooks(t *testing.T) {
 	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
 
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	require.NoError(t, err)
 	req.Header.Set("X-JWT", tk)
-	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
 	defer resp.Body.Close()
@@ -454,10 +444,10 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 	require.Nil(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
+	time.AfterFunc(duration, func() {
-		time.Sleep(duration)
 		log.Print("[TEST] terminate app")
 		cancel()
-	}()
+	})
+	rand.Seed(time.Now().UnixNano())
 	return app, ctx
 }

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )

backend/app/migrator/disqus_test.go

@@ -20,13 +20,13 @@ func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	d := Disqus{DataStore: &dataStore}
 	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
 	assert.Nil(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, adminUser)
 	assert.Nil(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 

backend/app/migrator/migrator.go

@@ -27,7 +27,7 @@ type Exporter interface {
 // Store defines minimal interface needed to export and import comments
 type Store interface {
 	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
 	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
@@ -42,6 +42,8 @@ type ImportParams struct {
 	SiteID    string
 }
 
+var adminUser = store.User{Admin: true}
+
 // ImportComments imports from given provider format and saves to store
 func ImportComments(p ImportParams) (int, error) {
 	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)

backend/app/migrator/migrator_test.go

@@ -4,11 +4,13 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+	"time"
 
 	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
@@ -25,7 +27,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
@@ -35,7 +37,7 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 4, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 4, len(last), "4 comments imported")
 }
@@ -51,7 +53,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/wordpress-test.xml",
@@ -61,7 +63,7 @@ func TestMigrator_ImportWordPress(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last("test", 10)
+	last, err := dataStore.Last("test", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
@@ -80,7 +82,7 @@ func TestMigrator_ImportNative(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := &service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -91,7 +93,7 @@ func TestMigrator_ImportNative(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 2, size)
 
-	last, err := dataStore.Last("radio-t", 10)
+	last, err := dataStore.Last("radio-t", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(last), "2 comments imported")
 }
@@ -100,7 +102,7 @@ func TestMigrator_ImportFailed(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	dataStore := &service.DataStore{Engine: b}
 	_, err = ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",

backend/app/migrator/native.go

@@ -49,7 +49,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	commentsCount := 0
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
-		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time", adminUser)
 		if e != nil {
 			return commentsCount, e
 		}

backend/app/migrator/native_test.go

@@ -40,25 +40,25 @@ func TestNative_Export(t *testing.T) {
 
 	dec := json.NewDecoder(strings.NewReader(c1))
 
-	meta := struct {
+	m := struct {
 		Version int                    `json:"version"`
 		Users   []service.UserMetaData `json:"users"`
 		Posts   []service.PostMetaData `json:"posts"`
 	}{}
 
-	require.NoError(t, dec.Decode(&meta), "decode meta")
+	require.NoError(t, dec.Decode(&m), "decode meta")
 
-	assert.Equal(t, 2, len(meta.Users))
+	assert.Equal(t, 2, len(m.Users))
-	assert.Equal(t, "user1", meta.Users[0].ID)
+	assert.Equal(t, "user1", m.Users[0].ID)
-	assert.Equal(t, false, meta.Users[0].Blocked.Status)
+	assert.Equal(t, false, m.Users[0].Blocked.Status)
-	assert.Equal(t, true, meta.Users[0].Verified)
+	assert.Equal(t, true, m.Users[0].Verified)
-	assert.Equal(t, "user2", meta.Users[1].ID)
+	assert.Equal(t, "user2", m.Users[1].ID)
-	assert.Equal(t, true, meta.Users[1].Blocked.Status)
+	assert.Equal(t, true, m.Users[1].Blocked.Status)
-	assert.Equal(t, false, meta.Users[1].Verified)
+	assert.Equal(t, false, m.Users[1].Verified)
 
-	assert.Equal(t, 1, len(meta.Posts))
+	assert.Equal(t, 1, len(m.Posts))
-	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
+	assert.Equal(t, "https://radio-t.com", m.Posts[0].URL)
-	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+	assert.Equal(t, true, m.Posts[0].ReadOnly)
 
 	comments := [3]store.Comment{}
 
@@ -77,12 +77,13 @@ func TestNative_Import(t *testing.T) {
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.Nil(t, err)
 	assert.Equal(t, 2, size)
 
-	comments, err := b.Last("radio-t", 10)
+	comments, err := b.Last("radio-t", 10, time.Time{}, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments))
 	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
@@ -106,7 +107,8 @@ func TestNative_ImportWrongVersion(t *testing.T) {
 	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	size, err := r.Import(strings.NewReader(inp), "radio-t")
 	assert.EqualError(t, err, "unexpected import file version 2")
 	assert.Equal(t, 0, size)
@@ -126,11 +128,12 @@ func TestNative_ImportManyWithError(t *testing.T) {
 	buf.WriteString("{}\n")
 
 	b := prep(t) // write some recs
-	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	b.AdminStore = admin.NewStaticStore("12345", []string{}, "")
+	r := Native{DataStore: b}
 	n, err := r.Import(buf, "radio-t")
 	assert.EqualError(t, err, "failed to save 2 comments")
 	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time", store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1200, len(comments))
 }
@@ -142,7 +145,7 @@ func prep(t *testing.T) *service.DataStore {
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
 	assert.Nil(t, err)
 
-	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	b := &service.DataStore{Engine: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 
 	comment := store.Comment{
 		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",

backend/app/migrator/wordpress.go

@@ -143,7 +143,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 							commentsCh <- commentFormatter.Format(c)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
-								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+								log.Printf("[DEBUG] processed %d comments", stats.inpComments)
 							}
 						}
 					}

backend/app/migrator/wordpress_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -17,17 +17,17 @@ import (
 
 func TestWordPress_Import(t *testing.T) {
 	siteID := "testWP"
-	defer os.Remove("/tmp/remark-test.db")
+	defer func() { _ = os.Remove("/tmp/remark-test.db") }()
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
 	assert.Nil(t, err, "create store")
 
-	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	wp := WordPress{DataStore: &dataStore}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.Nil(t, err)
 	assert.Equal(t, 3, size)
 
-	last, err := dataStore.Last(siteID, 10)
+	last, err := dataStore.Last(siteID, 10, time.Time{}, adminUser)
 	assert.Nil(t, err)
 	assert.Equal(t, 3, len(last), "3 comments imported")
 

backend/app/notify/notify.go

@@ -31,7 +31,7 @@ type Destination interface {
 
 // Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
-	Get(locator store.Locator, id string) (store.Comment, error)
+	Get(locator store.Locator, id string, user store.User) (store.Comment, error)
 }
 
 type request struct {
@@ -69,7 +69,7 @@ func (s *Service) Submit(comment store.Comment) {
 	}
 	parentComment := store.Comment{}
 	if s.dataService != nil {
-		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
+		if p, err := s.dataService.Get(comment.Locator, comment.ParentID, store.User{}); err == nil {
 			parentComment = p
 		}
 	}

backend/app/notify/notify_test.go

@@ -146,7 +146,7 @@ func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v",
 
 type mockStore struct{ data map[string]store.Comment }
 
-func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+func (m *mockStore) Get(_ store.Locator, id string, user store.User) (store.Comment, error) {
 	res, ok := m.data[id]
 	if !ok {
 		return store.Comment{}, errors.New("no such id")

backend/app/notify/telegram.go

@@ -113,10 +113,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/rest/api/admin.go

@@ -15,35 +15,29 @@ import (
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
-	dataService   *service.DataStore
+	dataService   adminStore
 	cache         cache.LoadingCache
 	authenticator *auth.Service
 	readOnlyAge   int
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
+type adminStore interface {
-	router := chi.NewRouter()
+	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error
-	router.Use(middlewares...)
+	DeleteUser(siteID string, userID string, mode store.DeleteMode) error
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
-	router.Put("/user/{userid}", a.setBlockCtrl)
+	IsBlocked(siteID string, userID string) bool
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
+	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
+	Blocked(siteID string) ([]store.BlockedUser, error)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
+	SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error)
-	router.Put("/pin/{id}", a.setPinCtrl)
+	SetVerified(siteID string, userID string, status bool) error
-	router.Get("/blocked", a.blockedUsersCtrl)
+	SetReadOnly(locator store.Locator, status bool) error
-	router.Put("/readonly", a.setReadOnlyCtrl)
+	SetPin(locator store.Locator, commentID string, status bool) error
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
 }
 
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
@@ -70,7 +64,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
 
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
+	if err := a.dataService.DeleteUser(siteID, userID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user", rest.ErrInternal)
 		return
 	}
@@ -86,7 +80,7 @@ func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
 
-	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0, rest.GetUserOrEmpty(r))
 	if err != nil || len(ucomments) == 0 {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info", rest.ErrInternal)
 		return
@@ -115,7 +109,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID, store.HardDelete); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
@@ -150,6 +144,13 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status", rest.ErrActionRejected)
 		return
 	}
+
+	// delete comments for permanently blocked user.
+	if blockStatus && ttl == time.Duration(0) {
+		if err := a.dataService.DeleteUser(siteID, userID, store.SoftDelete); err != nil {
+			log.Printf("[WARN] can't delete comments for blocked user %s on site %s, %v", userID, siteID, err)
+		}
+	}
 	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
@@ -237,42 +238,3 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

backend/app/rest/api/admin_test.go

@@ -58,12 +58,11 @@ func TestAdmin_Delete(t *testing.T) {
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 
 	// delete a comment
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err = client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -134,13 +133,12 @@ func TestAdmin_Title(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
@@ -170,38 +168,37 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	_, err = srv.DataService.Create(c3)
 	assert.NoError(t, err)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
+	cmntWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
+	err = json.Unmarshal([]byte(res), &cmntWithInfo)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
+	assert.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
 
 	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
+	assert.Equal(t, id1, cmntWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
+	assert.Equal(t, "o test test #1", cmntWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
+	assert.False(t, cmntWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
+	t.Logf("%+v", cmntWithInfo.Comments[0].User)
 
 	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
+	assert.True(t, cmntWithInfo.Comments[1].Deleted)
 
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
+	assert.True(t, cmntWithInfo.Comments[2].Deleted)
 }
 
 func TestAdmin_Pin(t *testing.T) {
@@ -221,6 +218,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
+		requireAdminOnly(t, req)
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
@@ -251,26 +249,27 @@ func TestAdmin_Block(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+	makeTwoComments := func() {
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+		c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+		c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+			URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
 
-	_, err := srv.DataService.Create(c1)
+		_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
+		require.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
+		_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
+		require.Nil(t, err)
+	}
 
 	block := func(val int, ttl string) (code int, body []byte) {
-		client := http.Client{}
 		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
 		if ttl != "" {
 			url = url + "&ttl=" + ttl
 		}
 		req, e := http.NewRequest(http.MethodPut, url, nil)
 		assert.Nil(t, e)
-		req.SetBasicAuth("admin", "password")
+		requireAdminOnly(t, req)
-		resp, e := client.Do(req)
+		resp, e := sendReq(t, req, adminUmputunToken)
 		require.Nil(t, e)
 		body, e = ioutil.ReadAll(resp.Body)
 		assert.Nil(t, e)
@@ -278,25 +277,49 @@ func TestAdmin_Block(t *testing.T) {
 		return resp.StatusCode, body
 	}
 
+	makeTwoComments()
+
 	// block permanently
 	code, body := block(1, "")
 	require.Equal(t, 200, code)
 	j := R.JSON{}
-	err = json.Unmarshal(body, &j)
+	err := json.Unmarshal(body, &j)
 	assert.Nil(t, err)
 	assert.Equal(t, "user1", j["user_id"])
 	assert.Equal(t, true, j["block"])
 	assert.Equal(t, "radio-t", j["site_id"])
 
+	assert.True(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
+
+	// get last to confirm one comment deleted
+	bodyStr, code := get(t, ts.URL+"/api/v1/last/10?site=radio-t")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	assert.NoError(t, json.Unmarshal([]byte(bodyStr), &pi))
+	assert.Equal(t, 1, len(pi), "last status updated, one comment left")
+
+	// check if count call has one comment left
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	body, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	pi = []store.PostInfo{}
+	err = json.Unmarshal(body, &pi)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1}}), pi)
+
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments := commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[0].Text, "permanent block clear comment")
-	assert.True(t, comments.Comments[0].Deleted)
+	assert.True(t, comments.Comments[0].Deleted, "permanent block set deleted comment's status")
 
+	// unblock
 	code, body = block(-1, "")
 	require.Equal(t, 200, code)
 	err = json.Unmarshal(body, &j)
@@ -304,78 +327,91 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, false, j["block"])
 
 	// block with ttl
+	makeTwoComments()
 	code, _ = block(1, "50ms")
 	require.Equal(t, 200, code)
 
+	// get as regular user
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "test test #1", comments.Comments[2].Text, "comment not removed and not cleared")
-	assert.True(t, comments.Comments[0].Deleted)
+	assert.False(t, comments.Comments[2].Deleted, "not deleted")
 
-	srv.Cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
+	srv.pubRest.cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
 	time.Sleep(50 * time.Millisecond)
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
 	comments = commentsWithInfo{}
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, 4, len(comments.Comments), "should have 4 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
+	assert.Equal(t, "test test #1", comments.Comments[2].Text, "restored")
-	assert.False(t, comments.Comments[0].Deleted)
+	assert.False(t, comments.Comments[2].Deleted)
+
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user1"))
+	assert.False(t, srv.adminRest.dataService.IsBlocked("radio-t", "user2"))
 }
 
 func TestAdmin_BlockedList(t *testing.T) {
-	ts, _, teardown := startupT(t)
+	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2 name", ID: "user2"}}
+
+	// write comments for user1 and user2
+	_, err := srv.DataService.Create(c1)
+	assert.Nil(t, err)
+	_, err = srv.DataService.Create(c2)
+	assert.Nil(t, err)
 
 	// block user1
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err := sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	res, err := client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(users), "two users blocked")
 	assert.Equal(t, "user1", users[0].ID)
+	assert.Equal(t, "user1 name", users[0].Name)
 	assert.Equal(t, "user2", users[1].ID)
-
+	assert.Equal(t, "user2 name", users[1].Name)
+	t.Logf("%+v", users)
 	time.Sleep(50 * time.Millisecond)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	res, err = sendReq(t, req, adminUmputunToken)
-	res, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(users), "one user left blocked")
-
 }
 
 func TestAdmin_ReadOnly(t *testing.T) {
@@ -396,15 +432,15 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err := sendReq(t, req, "") // non-admin user
-	resp, err := client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
@@ -416,20 +452,18 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err := json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
-	assert.Nil(t, err)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
@@ -440,10 +474,9 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err = json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
-	assert.Nil(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
@@ -451,15 +484,13 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.NotNil(t, err)
@@ -488,30 +519,27 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, info.ReadOnly, "ro by age")
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	resp, err = client.Do(req)
+	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 }
@@ -532,13 +560,13 @@ func TestAdmin_Verify(t *testing.T) {
 	verified := srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	_, err = client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.True(t, verified)
 
@@ -554,9 +582,9 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
+	resp, err = sendReq(t, req, adminUmputunToken)
-	_, err = client.Do(req)
+	require.NoError(t, err)
-	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
@@ -601,12 +629,11 @@ func TestAdmin_ExportFile(t *testing.T) {
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
+	require.NoError(t, err)
-	req.SetBasicAuth("admin", "password")
+	requireAdminOnly(t, req)
-	resp, err := client.Do(req)
+	resp, err := sendReq(t, req, adminUmputunToken)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	assert.Equal(t, 200, resp.StatusCode)
 	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
@@ -634,7 +661,7 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	assert.Nil(t, err)
 
-	comments, err := srv.DataService.User("radio-t", "user1", 0, 0)
+	comments, err := srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments), "a comment for user1")
 
@@ -665,12 +692,13 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
+
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	_, err = srv.DataService.User("radio-t", "user1", 0, 0)
+	_, err = srv.DataService.User("radio-t", "user1", 0, 0, store.User{})
 	assert.EqualError(t, err, "no comments for user user1 in store")
 }
 

backend/app/rest/api/migrator.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -40,15 +39,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {

backend/app/rest/api/migrator_test.go

@@ -6,38 +6,27 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/migrator"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -52,7 +41,7 @@ func TestMigrator_Import(t *testing.T) {
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
@@ -68,7 +57,7 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=radio-t&provider=native", contentType, bodyBuf)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
@@ -80,13 +69,13 @@ func TestMigrator_ImportForm(t *testing.T) {
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress", r)
 	assert.Nil(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
@@ -102,14 +91,14 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
@@ -117,7 +106,7 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
@@ -127,7 +116,8 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -135,7 +125,8 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err = client.Do(req)
@@ -145,7 +136,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
@@ -155,7 +146,8 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
 	resp, err := client.Do(req)
@@ -163,7 +155,8 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t&timeout=100ms", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
@@ -174,7 +167,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 }
 
 func TestMigrator_Export(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
@@ -182,7 +175,7 @@ func TestMigrator_Export(t *testing.T) {
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
@@ -191,7 +184,7 @@ func TestMigrator_Export(t *testing.T) {
 	waitForImportCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -208,7 +201,7 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -222,7 +215,7 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t", nil)
 	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
@@ -231,7 +224,8 @@ func TestMigrator_Export(t *testing.T) {
 
 func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t", nil)
+	require.NoError(t, err)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err := client.Do(req)
@@ -243,38 +237,6 @@ func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
 	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
 }
 
-func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
-	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc := &Migrator{
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
-		Cache:             &cache.Nop{},
-		KeyStore:          adminStore,
-	}
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	teardown = func() {
-		ts.Close()
-		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
-	}
-	return ts, teardown
-}
-
 var xmlTestWP = `
 <?xml version="1.0" encoding="UTF-8" ?>
 <rss version="2.0"

backend/app/rest/api/rest.go

@@ -5,9 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
 	"strings"
 	"sync"
 	"time"
@@ -46,6 +44,7 @@ type Rest struct {
 	Migrator         *Migrator
 	NotifyService    *notify.Service
 	ImageService     *image.Service
+	Streamer         *Streamer
 
 	WebRoot         string
 	RemarkURL       string
@@ -56,13 +55,17 @@ type Rest struct {
 		Critical int
 	}
 	UpdateLimiter float64
+	EmojiEnabled  bool
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
 	httpServer  *http.Server
 	lock        sync.Mutex
 
-	adminService admin
+	pubRest   public
+	privRest  private
+	adminRest admin
+	rssRest   rss
 }
 
 const hardBodyLimit = 1024 * 64 // limit size of body
@@ -159,24 +162,17 @@ func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
 		Addr:              fmt.Sprintf(":%d", port),
 		Handler:           router,
 		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
+		// WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
-		IdleTimeout:       30 * time.Second,
+		IdleTimeout: 30 * time.Second,
 	}
 }
 
 func (s *Rest) routes() chi.Router {
 	router := chi.NewRouter()
-	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
 	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
 
-	s.adminService = admin{
+	s.pubRest, s.privRest, s.adminRest, s.rssRest = s.controllerGroups() // assign controllers for groups
-		dataService:   s.DataService,
-		migrator:      s.Migrator,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
 
 	corsMiddleware := cors.New(cors.Options{
 		AllowedOrigins:   []string{"*"},
@@ -189,16 +185,18 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
+		r.Use(middleware.Timeout(5 * time.Second))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
 	router.Group(func(r chi.Router) {
+		r.Use(middleware.Timeout(5 * time.Second))
 		r.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)), middleware.NoCache)
 		r.Mount("/avatar", avatarHandler)
 	})
@@ -209,6 +207,7 @@ func (s *Rest) routes() chi.Router {
 	router.Route("/api/v1", func(rapi chi.Router) {
 
 		rapi.Group(func(rava chi.Router) {
+			rava.Use(middleware.Timeout(5 * time.Second))
 			rava.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
 			rava.Use(middleware.NoCache)
 			rava.Mount("/avatar", avatarHandler)
@@ -216,139 +215,209 @@ func (s *Rest) routes() chi.Router {
 
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
-			ropen.Use(middleware.NoCache)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
 			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
+			ropen.Get("/find", s.pubRest.findCommentsCtrl)
-			ropen.Get("/info", s.infoCtrl)
+			ropen.Get("/id/{id}", s.pubRest.commentByIDCtrl)
-			ropen.Mount("/img", s.ImageProxy.Routes())
+			ropen.Get("/comments", s.pubRest.findUserCommentsCtrl)
-			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Get("/last/{limit}", s.pubRest.lastCommentsCtrl)
+			ropen.Get("/count", s.pubRest.countCtrl)
+			ropen.Post("/counts", s.pubRest.countMultiCtrl)
+			ropen.Get("/list", s.pubRest.listCtrl)
+			ropen.Post("/preview", s.pubRest.previewCommentCtrl)
+			ropen.Get("/info", s.pubRest.infoCtrl)
+			ropen.Get("/img", s.ImageProxy.Handler)
+
+			ropen.Route("/rss", func(rrss chi.Router) {
+				rrss.Get("/post", s.rssRest.postCommentsCtrl)
+				rrss.Get("/site", s.rssRest.siteCommentsCtrl)
+				rrss.Get("/reply", s.rssRest.repliesCtrl)
+			})
+
+		})
+
+		// open routes, streams, no send timeout
+		rapi.Route("/stream", func(rstream chi.Router) {
+			rstream.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			rstream.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
+			rstream.Get("/info", s.pubRest.infoStreamCtrl)
+			rstream.Get("/last", s.pubRest.lastCommentsStreamCtrl)
 		})
 
 		// open routes, cached
 		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(middleware.Timeout(30 * time.Second))
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+			ropen.Get("/picture/{user}/{id}", s.pubRest.loadPictureCtrl)
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
-			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(middleware.Timeout(30 * time.Second))
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(authMiddleware.Auth, middleware.NoCache, logInfoWithBody)
-			rauth.Use(middleware.NoCache)
+			rauth.Get("/user", s.privRest.userInfoCtrl)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+			rauth.Get("/userdata", s.privRest.userAllDataCtrl)
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+		})
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
 
-			// admin routes, admin users only
+		// admin routes, require auth and admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(middleware.Timeout(30 * time.Second))
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminRest.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminRest.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminRest.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminRest.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminRest.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminRest.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminRest.setPinCtrl)
+			radmin.Get("/blocked", s.adminRest.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminRest.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminRest.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminRest.migrator.exportCtrl)
+			radmin.Post("/import", s.adminRest.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminRest.migrator.importFormCtrl)
+			radmin.Get("/import/wait", s.adminRest.migrator.importWaitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
+			rauth.Use(middleware.Timeout(10 * time.Second))
-			if s.UpdateLimiter > 0 {
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
 			rauth.Use(authMiddleware.Auth)
 			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
+			rauth.Put("/comment/{id}", s.privRest.updateCommentCtrl)
-			rauth.Post("/comment", s.createCommentCtrl)
+			rauth.Post("/comment", s.privRest.createCommentCtrl)
-			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
+			rauth.With(rejectAnonUser).Put("/vote/{id}", s.privRest.voteCtrl)
-			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.privRest.deleteMeCtrl)
 		})
 
+		// protected routes, anonymous rejected
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
+			rauth.Use(middleware.Timeout(10 * time.Second))
-			if s.UpdateLimiter > 0 {
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
-				lmt = s.UpdateLimiter
+			rauth.Use(authMiddleware.Auth, rejectAnonUser)
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+			rauth.Post("/picture", s.privRest.savePictureCtrl)
 		})
 
 	})
 
-	// respond to /robots.txt with the list of allowed paths
+	// open routes on root level
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
+	router.Group(func(rroot chi.Router) {
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		rroot.Use(middleware.Timeout(10 * time.Second))
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		rroot.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil)))
-				"/img", "/avatar", "/picture"}
+		rroot.Get("/index.html", s.pubRest.getStartedCtrl)
-			for i := range allowed {
+		rroot.Get("/robots.txt", s.pubRest.robotsCtrl)
-				allowed[i] = "Allow: /api/v1" + allowed[i]
+	})
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
 
 	// file server for static content from /web
 	addFileServer(router, "/web", http.Dir(s.WebRoot))
 	return router
 }
 
-func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
+func (s *Rest) controllerGroups() (public, private, admin, rss) {
 
-	res = s.adminService.alterComments(comments, r) // apply admin's alteration
+	pubGrp := public{
-
+		dataService:      s.DataService,
-	// prepare vote info for client view
+		cache:            s.Cache,
-	vote := func(c store.Comment, r *http.Request) store.Comment {
+		imageService:     s.ImageService,
-
+		commentFormatter: s.CommentFormatter,
-		c.Vote = 0 // default is "none" (not voted)
+		readOnlyAge:      s.ReadOnlyAge,
-
+		webRoot:          s.WebRoot,
-		user, err := rest.GetUserInfo(r)
+		streamer:         s.Streamer,
-		if err != nil {
-			c.Votes = nil // hide voters list and don't set Vote for non-authed user
-			return c
-		}
-
-		if v, ok := c.Votes[user.ID]; ok {
-			if v {
-				c.Vote = 1
-			} else {
-				c.Vote = -1
-			}
-		}
-
-		c.Votes = nil // hide voters list
-		return c
 	}
 
-	for i, c := range res {
+	privGrp := private{
-		c = vote(c, r)
+		dataService:      s.DataService,
-		res[i] = c
+		cache:            s.Cache,
+		imageService:     s.ImageService,
+		commentFormatter: s.CommentFormatter,
+		readOnlyAge:      s.ReadOnlyAge,
+		authenticator:    s.Authenticator,
+		notifyService:    s.NotifyService,
+		remarkURL:        s.RemarkURL,
 	}
 
-	return res
+	admGrp := admin{
+		dataService:   s.DataService,
+		migrator:      s.Migrator,
+		cache:         s.Cache,
+		authenticator: s.Authenticator,
+		readOnlyAge:   s.ReadOnlyAge,
+	}
+
+	rssGrp := rss{
+		dataService: s.DataService,
+		cache:       s.Cache,
+	}
+
+	return pubGrp, privGrp, admGrp, rssGrp
+}
+
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
+// GET /config?site=siteID - returns configuration
+func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	admins, _ := s.DataService.AdminStore.Admins(siteID)
+	emails, _ := s.DataService.AdminStore.Email(siteID)
+
+	cnf := struct {
+		Version        string   `json:"version"`
+		EditDuration   int      `json:"edit_duration"`
+		MaxCommentSize int      `json:"max_comment_size"`
+		Admins         []string `json:"admins"`
+		AdminEmail     string   `json:"admin_email"`
+		Auth           []string `json:"auth_providers"`
+		LowScore       int      `json:"low_score"`
+		CriticalScore  int      `json:"critical_score"`
+		PositiveScore  bool     `json:"positive_score"`
+		ReadOnlyAge    int      `json:"readonly_age"`
+		MaxImageSize   int      `json:"max_image_size"`
+		EmojiEnabled   bool     `json:"emoji_enabled"`
+	}{
+		Version:        s.Version,
+		EditDuration:   int(s.DataService.EditDuration.Seconds()),
+		MaxCommentSize: s.DataService.MaxCommentSize,
+		Admins:         admins,
+		AdminEmail:     emails,
+		LowScore:       s.ScoreThresholds.Low,
+		CriticalScore:  s.ScoreThresholds.Critical,
+		PositiveScore:  s.DataService.PositiveScore,
+		ReadOnlyAge:    s.ReadOnlyAge,
+		MaxImageSize:   s.ImageService.Store.SizeLimit(),
+		EmojiEnabled:   s.EmojiEnabled,
+	}
+
+	cnf.Auth = []string{}
+	for _, ap := range s.Authenticator.Providers() {
+		cnf.Auth = append(cnf.Auth, ap.Name())
+	}
+
+	if cnf.Admins == nil { // prevent json serialization to nil
+		cnf.Admins = []string{}
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, cnf)
 }
 
 // serves static files from /web or embedded by statik
@@ -374,15 +443,15 @@ func addFileServer(r chi.Router, path string, root http.FileSystem) {
 	}
 	path += "*"
 
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil)), middleware.Timeout(10*time.Second)).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		Get(path, func(w http.ResponseWriter, r *http.Request) {
 			// don't show dirs, just serve files
 			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
 				http.NotFound(w, r)
 				return
 			}
 			webFS.ServeHTTP(w, r)
-		}))
+		})
 }
 
 func encodeJSONWithHTML(v interface{}) ([]byte, error) {
@@ -448,3 +517,28 @@ func rejectAnonUser(next http.Handler) http.Handler {
 	}
 	return http.HandlerFunc(fn)
 }
+
+func parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}

backend/app/rest/api/rest_private.go

@@ -9,22 +9,49 @@ import (
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
+	"github.com/umputun/remark/backend/app/notify"
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
+type private struct {
+	dataService      privStore
+	cache            cache.LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	notifyService    *notify.Service
+	authenticator    *auth.Service
+	remarkURL        string
+}
+
+type privStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	EditComment(locator store.Locator, commentID string, req service.EditRequest) (comment store.Comment, err error)
+	Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	ValidateComment(c *store.Comment) error
+	IsVerified(siteID string, userID string) bool
+	IsReadOnly(locator store.Locator) bool
+	IsBlocked(siteID string, userID string) bool
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+}
+
 // POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
@@ -39,14 +66,14 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
 
 	comment.Orig = comment.Text // original comment text, prior to md render
-	if err := s.DataService.ValidateComment(&comment); err != nil {
+	if err := s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 
 	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
+	if s.dataService.IsBlocked(comment.Locator.SiteID, comment.User.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
@@ -56,7 +83,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	id, err := s.DataService.Create(comment)
+	id, err := s.dataService.Create(comment)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
@@ -66,17 +93,17 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// DataService modifies comment
+	// dataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
+	finalComment, err := s.dataService.Get(comment.Locator, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment", rest.ErrInternal)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+	s.cache.Flush(cache.Flusher(comment.Locator.SiteID).
 		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
 
-	if s.NotifyService != nil {
+	if s.notifyService != nil {
-		s.NotifyService.Submit(finalComment)
+		s.notifyService.Submit(finalComment)
 	}
 
 	log.Printf("[DEBUG] created commend %+v", finalComment)
@@ -86,7 +113,7 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	edit := struct {
 		Text    string
@@ -107,7 +134,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 
 	var currComment store.Comment
 	var err error
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
+	if currComment, err = s.dataService.Get(locator, id, rest.GetUserOrEmpty(r)); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment", rest.ErrCommentNotFound)
 		return
 	}
@@ -119,40 +146,40 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.CommentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,
 	}
 
-	res, err := s.DataService.EditComment(locator, id, editReq)
+	res, err := s.dataService.EditComment(locator, id, editReq)
 	if err == service.ErrRestrictedWordsFound {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
 	if err != nil {
-		code := s.parseError(err, rest.ErrCommentRejected)
+		code := parseError(err, rest.ErrCommentRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
 		return
 	}
 
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
 // GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
+		user.Verified = s.dataService.IsVerified(siteID, user.ID)
 	}
 
 	render.JSON(w, r, user)
 }
 
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
@@ -166,23 +193,23 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// check if user blocked
-	if s.adminService.checkBlocked(locator.SiteID, user) {
+	if s.dataService.IsBlocked(locator.SiteID, user.ID) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked", rest.ErrUserBlocked)
 		return
 	}
 
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
+	comment, err := s.dataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		code := s.parseError(err, rest.ErrVoteRejected)
+		code := parseError(err, rest.ErrVoteRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
-	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
+	s.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
 	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	user := rest.MustGetUserInfo(r)
 	userB, err := json.Marshal(&user)
@@ -213,7 +240,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		comments, errUser := s.dataService.User(siteID, user.ID, 100, i*100, rest.GetUserOrEmpty(r))
 		if errUser != nil {
 			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
@@ -240,7 +267,7 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 // POST /deleteme?site_id=site - requesting delete of all user info
 // makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 	siteID := r.URL.Query().Get("site")
 
@@ -260,18 +287,18 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
-	tokenStr, err := s.Authenticator.TokenService().Token(claims)
+	tokenStr, err := s.authenticator.TokenService().Token(claims)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token", rest.ErrInternal)
 		return
 	}
 
-	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
+	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.remarkURL, tokenStr)
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
 // POST /image - save image with form request
-func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *private) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	user := rest.MustGetUserInfo(r)
 
 	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
@@ -286,7 +313,7 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = file.Close() }()
 
-	id, err := s.ImageService.Save(header.Filename, user.ID, file)
+	id, err := s.imageService.Save(header.Filename, user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return
@@ -295,37 +322,12 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"id": id})
 }
 
-func (s *Rest) isReadOnly(locator store.Locator) bool {
+func (s *private) isReadOnly(locator store.Locator) bool {
-	if s.ReadOnlyAge > 0 {
+	if s.readOnlyAge > 0 {
 		// check RO by age
-		if info, e := s.DataService.Info(locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+		if info, e := s.dataService.Info(locator, s.readOnlyAge); e == nil && info.ReadOnly {
 			return true
 		}
 	}
-	return s.DataService.IsReadOnly(locator) // ro manually
+	return s.dataService.IsReadOnly(locator) // ro manually
-}
-
-func (s *Rest) parseError(err error, defaultCode int) (code int) {
-	code = defaultCode
-
-	switch {
-	// voting errors
-	case strings.Contains(err.Error(), "can not vote for his own comment"):
-		code = rest.ErrVoteSelf
-	case strings.Contains(err.Error(), "already voted for"):
-		code = rest.ErrVoteDbl
-	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
-		code = rest.ErrVoteMax
-	case strings.Contains(err.Error(), "minimal score reached for comment"):
-		code = rest.ErrVoteMinScore
-
-	// edit errors
-	case strings.HasPrefix(err.Error(), "too late to edit"):
-		code = rest.ErrCommentEditExpired
-	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
-		code = rest.ErrCommentEditChanged
-
-	}
-
-	return code
 }

backend/app/rest/api/rest_private_test.go

@@ -11,17 +11,14 @@ import (
 	"mime/multipart"
 	"net/http"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/rest"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/image"
@@ -63,7 +60,7 @@ func TestRest_CreateOldPost(t *testing.T) {
 	_, err := srv.DataService.Create(old)
 	assert.Nil(t, err)
 
-	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time")
+	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time", store.User{})
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments))
 
@@ -620,16 +617,21 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	}()
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
-	svc.ImageService = &image.Service{
+	imageService := svc.ImageService
-		Store: &image.FileSystem{
+	imageService.Store = &image.FileSystem{
-			Staging:  "/tmp/remark42/images.staging",
+		Staging:  "/tmp/remark42/images.staging",
-			Location: "/tmp/remark42/images",
+		Location: "/tmp/remark42/images",
-			MaxSize:  2000,
+		MaxSize:  2000,
-		},
-		TTL: time.Millisecond * 100,
 	}
-	svc.DataService.EditDuration = time.Millisecond * 100
+	imageService.TTL = 100 * time.Millisecond
-	svc.DataService.ImageService = svc.ImageService
+
+	svc.privRest.imageService = imageService
+	svc.ImageService = imageService
+
+	dataService := svc.DataService
+	dataService.EditDuration = time.Millisecond * 100
+	dataService.ImageService = svc.ImageService
+	svc.privRest.dataService = dataService
 
 	uploadPicture := func(file string) (id string) {
 		bodyBuf := &bytes.Buffer{}
@@ -683,26 +685,3 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	_, err = os.Stat("/tmp/remark42/images/" + id3)
 	assert.NoError(t, err, "moved from staging")
 }
-
-func TestRest_parseError(t *testing.T) {
-	tbl := []struct {
-		err error
-		res int
-	}{
-		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
-		{errors.New("already voted for"), rest.ErrVoteDbl},
-		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
-		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
-		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
-		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
-		{errors.New("blah blah"), rest.ErrInternal},
-	}
-
-	svc := Rest{}
-	for n, tt := range tbl {
-		t.Run(strconv.Itoa(n), func(t *testing.T) {
-			res := svc.parseError(tt.err, rest.ErrInternal)
-			assert.Equal(t, tt.res, res)
-		})
-	}
-}

backend/app/rest/api/rest_public.go

@@ -4,51 +4,95 @@ import (
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
+type public struct {
+	dataService      pubStore
+	cache            cache.LoadingCache
+	readOnlyAge      int
+	commentFormatter *store.CommentFormatter
+	imageService     *image.Service
+	streamer         *Streamer
+	webRoot          string
+}
+
+type pubStore interface {
+	Create(comment store.Comment) (commentID string, err error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
+	UserCount(siteID, userID string) (int, error)
+	Count(locator store.Locator) (int, error)
+	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
+	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
+
+	ValidateComment(c *store.Comment) error
+	IsReadOnly(locator store.Locator) bool
+	Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
+}
+
+// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec
 // find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	sort := r.URL.Query().Get("sort")
 	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
 		sort = "+" + sort[1:]
 	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
+
+	view := r.URL.Query().Get("view")
+	since, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
+		return
+	}
+	format := r.URL.Query().Get("format")
+	if format == "tree" {
+		since = time.Time{} // since doesn't make sense for tree
+	}
+
+	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v", locator, sort, format, since)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, sort)
+		comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
 		if e != nil {
 			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
-		maskedComments := s.alterComments(comments, r)
+		comments = s.applyView(comments, view)
 		var b []byte
-		switch r.URL.Query().Get("format") {
+		switch format {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(comments, sort, s.readOnlyAge)
 			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*rest.Node{}
+				tree.Nodes = []*service.Node{}
 			}
-			if s.DataService.IsReadOnly(locator) {
+			if s.dataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
 			b, e = encodeJSONWithHTML(tree)
 		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
+			withInfo := commentsWithInfo{Comments: comments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
+			if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
 				withInfo.Info = info
 			}
 			b, e = encodeJSONWithHTML(withInfo)
@@ -67,7 +111,7 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	comment := store.Comment{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment", rest.ErrDecode)
@@ -81,23 +125,23 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	comment.User = user
 	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
+	if err = s.dataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
 
-	comment = s.CommentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment)
 	comment.Sanitize()
 	render.HTML(w, r, comment.Text)
 }
 
 // GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
+		info, e := s.dataService.Info(locator, s.readOnlyAge)
 		if e != nil {
 			return nil, e
 		}
@@ -114,8 +158,53 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
+// GET /stream/info?site=siteID&url=post-url&since=unix_ts_msec - get info stream about the post
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) infoStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+	log.Printf("[DEBUG] start stream for %+v, timeout=%v, refresh=%v", locator, s.streamer.TimeOut, s.streamer.Refresh)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
+	fn := func() steamEventFn {
+		lastTS := sinceTs
+		lastCount := 0
+
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				info, e := s.dataService.Info(locator, s.readOnlyAge)
+				if e != nil {
+					return nil, e
+				}
+				// cache update used as indication of post update. comparing lastTS for no-cache.
+				// removal won't update lastTS, count check will catch it.
+				if !lastTS.IsZero() && (info.LastTS != lastTS || info.Count != lastCount) {
+					upd = true
+				}
+				lastTS = info.LastTS
+				lastCount = info.Count
+				return encodeJSONWithHTML(info)
+			})
+			if err != nil {
+				return "info", data, false, err
+			}
+
+			return "info", data, upd, nil
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
+// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
+// limited with "since" param
+func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get last comments for %s", siteID)
 
@@ -124,13 +213,18 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		limit = 0
 	}
 
+	sinceTime, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
+		comments, e := s.dataService.Last(siteID, limit, sinceTime, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
 		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
 		return encodeJSONWithHTML(filterDeleted)
@@ -146,8 +240,47 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GET /stream/last?site=siteID&since=unix_ts_ms - stream of last comments last comments for the siteID, across all posts
+func (s *public) lastCommentsStreamCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] get last comments stream for %s", siteID)
+
+	sinceTs, err := s.parseSince(r)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
+		return
+	}
+	if sinceTs.IsZero() {
+		sinceTs = time.Now()
+	}
+
+	fn := func() steamEventFn {
+		sinceTime := sinceTs
+		return func() (event string, data []byte, upd bool, err error) {
+			key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
+			data, err = s.cache.Get(key, func() ([]byte, error) {
+				comments, e := s.dataService.Last(siteID, 1, sinceTime, rest.GetUserOrEmpty(r))
+				if e != nil {
+					return nil, e
+				}
+				if len(comments) > 0 {
+					sinceTime = comments[0].Timestamp
+					upd = true
+				}
+				sinceTime = time.Now()
+				return encodeJSONWithHTML(comments)
+			})
+			return "last", data, upd, err
+		}
+	}
+
+	if e := s.streamer.Activate(r.Context(), fn, w); e != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, e, "can't stream", rest.ErrInternal)
+	}
+}
+
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	id := chi.URLParam(r, "id")
 	siteID := r.URL.Query().Get("site")
@@ -155,12 +288,11 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
 
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
+	comment, err := s.dataService.Get(store.Locator{SiteID: siteID, URL: url}, id, rest.GetUserOrEmpty(r))
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
 		return
 	}
-	comment = s.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
 
 	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
@@ -169,7 +301,7 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
@@ -187,14 +319,13 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
 	key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
+		comments, e := s.dataService.User(siteID, userID, limit, 0, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
+		count, e := s.dataService.UserCount(siteID, userID)
 		if e != nil {
 			return nil, e
 		}
@@ -212,53 +343,10 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		PositiveScore  bool     `json:"positive_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-		MaxImageSize   int      `json:"max_image_size"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.DataService.AdminStore.Admins(siteID),
-		AdminEmail:     s.DataService.AdminStore.Email(siteID),
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		PositiveScore:  s.DataService.PositiveScore,
-		ReadOnlyAge:    s.ReadOnlyAge,
-		MaxImageSize:   s.ImageService.Store.SizeLimit(),
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers() {
-		cnf.Auth = append(cnf.Auth, ap.Name())
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
 // GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
+	count, err := s.dataService.Count(locator)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
 		return
@@ -267,7 +355,7 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // POST /counts?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	posts := []string{}
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
@@ -277,12 +365,12 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	h := sha1.Sum([]byte(k)) //nolint
+	h := sha1.Sum([]byte(k)) // nolint
 	sha := base64.URLEncoding.EncodeToString(h[:])
 
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
+		counts, e := s.dataService.Counts(siteID, posts)
 		if e != nil {
 			return nil, e
 		}
@@ -300,7 +388,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
 
 	siteID := r.URL.Query().Get("site")
 	limit, skip := 0, 0
@@ -313,8 +401,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
+		posts, e := s.dataService.List(siteID, limit, skip)
 		if e != nil {
 			return nil, e
 		}
@@ -327,12 +415,12 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
-		log.Printf("[WARN] can't render posts lits for site %s", siteID)
+		log.Printf("[WARN] can't render posts list for site %s", siteID)
 	}
 }
 
 // GET /picture/{user}/{id} - get picture
-func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 
 	imgContentType := func(img string) string {
 		img = strings.ToLower(img)
@@ -348,7 +436,7 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
-	imgRdr, size, err := s.ImageService.Load(id)
+	imgRdr, size, err := s.imageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
 		return
@@ -364,7 +452,11 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer imgRdr.Close()
+	defer func() {
+		if e := imgRdr.Close(); e != nil {
+			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
+		}
+	}()
 
 	w.Header().Set("Content-Type", imgContentType(id))
 	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
@@ -373,3 +465,50 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *public) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.webRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *public) robotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}
+
+func (s *public) applyView(comments []store.Comment, view string) []store.Comment {
+	if strings.EqualFold(view, "user") {
+		projection := make([]store.Comment, len(comments))
+		for i, c := range comments {
+			p := store.Comment{
+				ID:   c.ID,
+				User: c.User,
+			}
+			projection[i] = p
+		}
+		return projection
+	}
+	return comments
+}
+
+func (s *public) parseSince(r *http.Request) (time.Time, error) {
+	sinceTs := time.Time{}
+	if since := r.URL.Query().Get("since"); since != "" {
+		unixTS, e := strconv.ParseInt(since, 10, 64)
+		if e != nil {
+			return time.Time{}, errors.Wrap(e, "can't translate since parameter")
+		}
+		sinceTs = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
+	}
+	return sinceTs, nil
+}

backend/app/rest/api/rest_public_test.go

@@ -1,21 +1,25 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -37,6 +41,10 @@ func TestRest_Preview(t *testing.T) {
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
 	assert.Equal(t, "<p>test 123</p>\n", string(b))
+
+	resp, err = post(t, ts.URL+"/api/v1/preview", "bad")
+	assert.Nil(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_PreviewWithMD(t *testing.T) {
@@ -96,6 +104,8 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
 	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "<p>test test #1</p>\n", comments.Comments[0].Text)
+	assert.Equal(t, "<p>test test #2</p>\n", comments.Comments[1].Text)
 	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
 	assert.Equal(t, 2, comments.Info.Count)
 	assert.Equal(t, false, comments.Info.ReadOnly)
@@ -111,7 +121,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -137,7 +147,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
@@ -178,7 +188,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -186,7 +196,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
+	tree = service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -195,6 +205,42 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.False(t, tree.Info.ReadOnly, "post is writable")
 }
 
+func TestRest_FindUserView(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&view=user")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id1 := addComment(t, c1, ts)
+
+	c2 := store.Comment{Text: "test test #2", ParentID: id1,
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id2 := addComment(t, c2, ts)
+
+	assert.NotEqual(t, id1, id2)
+
+	// get sorted by +time with view=user
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time&view=user")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, id1, comments.Comments[0].ID)
+	assert.Equal(t, id2, comments.Comments[1].ID)
+	assert.Equal(t, "dev", comments.Comments[0].User.ID)
+	assert.Equal(t, "dev", comments.Comments[1].User.ID)
+	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.Equal(t, "", comments.Comments[1].Text)
+}
+
 func TestRest_Last(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
@@ -209,8 +255,11 @@ func TestRest_Last(t *testing.T) {
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
 
 	// add 3 comments
+	ts1 := time.Now().UnixNano() / 1000000
 	addComment(t, c1, ts)
 	id1 := addComment(t, c1, ts)
+	time.Sleep(10 * time.Millisecond)
+	ts2 := time.Now().UnixNano() / 1000000
 	id2 := addComment(t, c2, ts)
 
 	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
@@ -222,6 +271,23 @@ func TestRest_Last(t *testing.T) {
 	assert.Equal(t, id1, comments[1].ID)
 	assert.Equal(t, id2, comments[0].ID)
 
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts1))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	assert.Equal(t, id1, comments[1].ID)
+	assert.Equal(t, id2, comments[0].ID)
+
+	res, code = get(t, fmt.Sprintf("%s/api/v1/last/2?site=radio-t&since=%d", ts.URL, ts2))
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+	assert.Equal(t, id2, comments[0].ID)
+
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
@@ -243,6 +309,9 @@ func TestRest_Last(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
 	t.Logf("%+v", comments)
+
+	_, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t-BLAH")
+	assert.Equal(t, 500, code)
 }
 
 func TestRest_FindUserComments(t *testing.T) {
@@ -320,6 +389,9 @@ func TestRest_Count(t *testing.T) {
 	err = json.Unmarshal([]byte(body), &j)
 	assert.Nil(t, err)
 	assert.Equal(t, 2.0, j["count"])
+
+	_, code = get(t, ts.URL+"/api/v1/count?site=radio-t-BLAH&url=https://radio-t.com/blah1XXX")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_Counts(t *testing.T) {
@@ -349,6 +421,10 @@ func TestRest_Counts(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
 		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
+
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-XXX", `{}`)
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_List(t *testing.T) {
@@ -375,6 +451,9 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 2, pi[0].Count)
 	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
 	assert.Equal(t, 3, pi[1].Count)
+
+	_, code = get(t, ts.URL+"/api/v1/list?site=radio-t-BLAH")
+	assert.Equal(t, 400, code)
 }
 
 func TestRest_ListWithSkipAndLimit(t *testing.T) {
@@ -426,6 +505,7 @@ func TestRest_Config(t *testing.T) {
 	assert.False(t, j["positive_score"].(bool))
 	assert.Equal(t, 10., j["readonly_age"])
 	assert.Equal(t, 10000., j["max_image_size"])
+	assert.Equal(t, true, j["emoji_enabled"].(bool))
 	t.Logf("%+v", j)
 }
 
@@ -433,7 +513,7 @@ func TestRest_Info(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
 
 	user := store.User{ID: "user1", Name: "user name 1"}
 	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
@@ -466,6 +546,155 @@ func TestRest_Info(t *testing.T) {
 	assert.Equal(t, 400, code)
 }
 
+func TestRest_InfoStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	wg.Wait()
+
+	t.Logf(string(body))
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 records. each 2 lines +1 emty line")
+	assert.True(t, strings.Contains(recs[0+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[9*3+1], `"count":11`), recs[9])
+
+	_, code = get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah123")
+	assert.Equal(t, 500, code)
+}
+
+func TestRest_InfoStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+}
+
+func TestRest_InfoStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_InfoStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 5; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+			log.Printf("write #%d", i)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1", nil)
+	require.Nil(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	log.Print("start req")
+	r, err := client.Do(req)
+	log.Print("end req")
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 2*3, len(recs), "should have 2 events")
+	assert.True(t, strings.Contains(recs[0*3+1], `"count":2`), recs[0])
+	assert.True(t, strings.Contains(recs[1*3+1], `"count":3`), recs[1])
+}
+
+func TestRest_InfoStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 10; i++ {
+			time.Sleep(10 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	body, code := get(t, ts.URL+"/api/v1/stream/info?site=radio-t&url=https://radio-t.com/blah1&since=12345678")
+	assert.Equal(t, 200, code)
+	wg.Wait()
+
+	t.Logf(string(body))
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 11*3, len(recs), "include first record, total 11 records. each 2 lines +1 empty line")
+}
+
 func TestRest_Robots(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
@@ -476,3 +705,177 @@ func TestRest_Robots(t *testing.T) {
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
 		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }
+
+func TestRest_LastCommentsStream(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	require.Nil(t, err)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+	t.Logf("headers: %+v", r.Header)
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+	assert.Equal(t, "keep-alive", r.Header.Get("connection"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 9*3, len(recs), "9 events")
+	t.Logf("%s", string(body))
+	assert.True(t, strings.Contains(recs[1], `test 123`), recs[1])
+}
+
+func TestRest_LastCommentsStreamTimeout(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 450 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	st := time.Now()
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.True(t, time.Since(st) > time.Millisecond*450 && time.Since(st) < time.Millisecond*500, time.Since(st))
+}
+
+func TestRest_LastCommentsStreamCancel(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t", nil)
+	require.Nil(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(), 290*time.Millisecond)
+	defer cancel()
+	req = req.WithContext(ctx)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.EqualError(t, err, "context deadline exceeded")
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 2*3, len(recs), "2 events")
+	assert.True(t, strings.Contains(recs[0+1], `test 123`), recs[0+1])
+}
+
+func TestRest_LastCommentsStreamTooMany(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 1 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 300 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 10
+
+	postComment(t, ts.URL)
+
+	var errsCount int32
+	wg := sync.WaitGroup{}
+	wg.Add(20)
+	for i := 0; i < 20; i++ {
+		go func() {
+			_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+			if code == 429 {
+				atomic.AddInt32(&errsCount, 1)
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(10), atomic.LoadInt32(&errsCount), "10 streams rejected")
+
+	_, code := get(t, ts.URL+"/api/v1/stream/last?site=radio-t")
+	assert.Equal(t, 200, code, "all streams closed, good to go again")
+}
+
+func TestRest_LastCommentsStreamSince(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	srv.pubRest.readOnlyAge = 10000000 // make sure we don't hit read-only
+	srv.pubRest.streamer.Refresh = 10 * time.Millisecond
+	srv.pubRest.streamer.TimeOut = 500 * time.Millisecond
+	srv.pubRest.streamer.MaxActive = 100
+
+	postComment(t, ts.URL)
+
+	defer teardown()
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 1; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			postComment(t, ts.URL)
+		}
+	}()
+
+	client := http.Client{}
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/stream/last?site=radio-t&since=123456", nil)
+	require.Nil(t, err)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode)
+
+	wg.Wait()
+	t.Logf("headers: %+v", r.Header)
+	assert.Equal(t, "text/event-stream", r.Header.Get("content-type"))
+
+	recs := strings.Split(strings.TrimSuffix(string(body), "\n"), "\n")
+	require.Equal(t, 10*3, len(recs), "10 events, includes first record")
+	t.Logf("%v", recs)
+}
+
+func postComment(t *testing.T, url string) {
+	resp, e := post(t, url+"/api/v1/comment",
+		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+	require.Nil(t, e)
+	b, e := ioutil.ReadAll(resp.Body)
+	require.Nil(t, e)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+}

backend/app/rest/api/rest_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -19,6 +20,7 @@ import (
 	"github.com/go-pkgz/auth"
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
@@ -37,9 +39,16 @@ import (
 var testHTML = "/tmp/test-remark.html"
 var getStartedHTML = "/tmp/getstarted.html"
 
-var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." +
+	"eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIs" +
+	"InVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMT" +
+	"I3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
-var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9." +
+	"eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1" +
+	"NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0" +
+	"dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlI" +
+	"iwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.gAR_sZT7hTx7CNHByyrJQWMB5tAtoiISAiG8kes1IjA"
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
@@ -249,7 +258,32 @@ func Test_URLKeyWithUser(t *testing.T) {
 	}
 
 }
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}
+
 func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	// log.Setup(log.Out(ioutil.Discard))
+	log.Setup(log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 
 	testDb := fmt.Sprintf("/tmp/test-remark-%d.db", rand.Int31())
 	os.Remove(testDb)
@@ -263,14 +297,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 	memCache, err := cache.NewMemoryCache()
 	assert.NoError(t, err)
 
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	astore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
 	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
 
 	dataStore := &service.DataStore{
-		Interface:              b,
+		Engine:                 b,
 		EditDuration:           5 * time.Minute,
 		MaxCommentSize:         4000,
-		AdminStore:             adminStore,
+		AdminStore:             astore,
 		MaxVotes:               service.UnlimitedVotes,
 		RestrictedWordsMatcher: restrictedWordsMatcher,
 	}
@@ -303,8 +337,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 			NativeImporter:    &migrator.Native{DataStore: dataStore},
 			NativeExporter:    &migrator.Native{DataStore: dataStore},
 			Cache:             &cache.Nop{},
-			KeyStore:          adminStore,
+			KeyStore:          astore,
 		},
+		Streamer: &Streamer{
+			Refresh:   100 * time.Millisecond,
+			TimeOut:   5 * time.Second,
+			MaxActive: 100,
+		},
+		EmojiEnabled: true,
 	}
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
 
@@ -348,7 +388,7 @@ func get(t *testing.T, url string) (string, int) {
 	return string(body), r.StatusCode
 }
 
-func sendReq(t *testing.T, r *http.Request, token string) (*http.Response, error) {
+func sendReq(_ *testing.T, r *http.Request, token string) (*http.Response, error) {
 	client := http.Client{Timeout: 5 * time.Second}
 	if token != "" {
 		r.Header.Set("X-JWT", token)
@@ -409,3 +449,13 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}

backend/app/rest/api/rss.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
@@ -15,38 +14,40 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
+type rss struct {
+	dataService rssStore
+	cache       cache.LoadingCache
+}
+
+type rssStore interface {
+	Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error)
+	Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
+	Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
+	UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error)
+}
+
 const maxRssItems = 20
-const maxLastCommentsReply = 5000
 const maxReplyDuration = 31 * 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) postCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[DEBUG] get rss for post %+v", locator)
 
 	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
+		comments, e := s.dataService.Find(locator, "-time", rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
+		feed, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
-		rss, e := s.toRssFeed(locator.URL, comments, "post comments for "+r.URL.Query().Get("url"))
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -63,23 +64,22 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) siteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss for site %s", siteID)
 
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() ([]byte, error) {
+	data, err := s.cache.Get(key, func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
+		comments, e := s.dataService.Last(siteID, maxRssItems, time.Time{}, rest.GetUserOrEmpty(r))
 		if e != nil {
 			return nil, e
 		}
-		comments = s.alterComments(comments, r)
 
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
+		feed, e := s.toRssFeed(r.URL.Query().Get("site"), comments, "site comment for "+siteID)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -95,43 +95,24 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /rss/reply?user=userID&site=siteID
-func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
+func (s *rss) repliesCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := r.URL.Query().Get("user")
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
 
-	userName := ""
 	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
-	data, err := s.Cache.Get(key, func() (res []byte, e error) {
+	data, err := s.cache.Get(key, func() (res []byte, e error) {
-		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+
+		replies, userName, e := s.dataService.UserReplies(siteID, userID, maxRssItems, maxReplyDuration)
 		if e != nil {
 			return nil, errors.Wrap(e, "can't get last comments")
 		}
-		comments = s.alterComments(comments, r)
-		replies := []store.Comment{}
-		for _, c := range comments {
-			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
-				break
-			}
-			if c.User.ID != userID {
-				userName = c.User.Name
-			}
-			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
-				var pc store.Comment
-				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
-					return nil, errors.Wrap(e, "can't get parent comment")
-				}
-				if pc.User.ID == userID {
-					replies = append(replies, c)
-				}
-			}
-		}
 
-		rss, e := s.toRssFeed(siteID, replies, "replies to "+userName)
+		feed, e := s.toRssFeed(siteID, replies, "replies to "+userName)
 		if e != nil {
 			return nil, e
 		}
-		return []byte(rss), e
+		return []byte(feed), e
 	})
 
 	if err != nil {
@@ -146,7 +127,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (s *Rest) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
+func (s *rss) toRssFeed(url string, comments []store.Comment, description string) (string, error) {
 
 	if description == "" {
 		description = "comment updates"
@@ -175,9 +156,10 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment, description strin
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
+			parentComment, err := s.dataService.Get(c.Locator, c.ParentID, store.User{})
 			if err == nil {
 				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
+				f.Description = f.Description + "<blockquote><p>" + parentComment.Snippet(300) + "</p></blockquote>"
 			} else {
 				log.Printf("[WARN] failed to get info about parent comment, %s", err)
 			}

backend/app/rest/api/rss_test.go

@@ -161,7 +161,7 @@ func TestServer_RssWithReply(t *testing.T) {
 		    <item>
 		      <title>developer one &gt; developer one</title>
 		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
-		      <description>xyz test</description>
+              <description>xyz test&lt;blockquote&gt;&lt;p&gt;test 123&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>developer one</author>
               <guid>comment-id-2</guid>
 		      <pubDate>%s</pubDate>
@@ -242,12 +242,12 @@ func TestServer_RssReplies(t *testing.T) {
 	    <channel>
 	        <title>Remark42 comments</title>
 	        <link>radio-t</link>
-	        <description>replies to user2</description>
+	        <description>replies to user1</description>
 	        <pubDate>%s</pubDate>
 	        <item>
 		      <title>user3 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
-		      <description>reply to c1 from user3</description>
+		      <description>reply to c1 from user3&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user3</author>
               <guid>comment-3</guid> 
 		      <pubDate>%s</pubDate>
@@ -255,7 +255,7 @@ func TestServer_RssReplies(t *testing.T) {
 			<item>
 		      <title>user2 &gt; user1</title>
 		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
-		      <description>reply to c1 from user2</description>
+		      <description>reply to c1 from user2&lt;blockquote&gt;&lt;p&gt;c1&lt;/p&gt;&lt;/blockquote&gt;</description>
 		      <author>user2</author>
               <guid>comment-2</guid> 
 		      <pubDate>%s</pubDate>

backend/app/rest/api/ssl.go

@@ -86,7 +86,7 @@ func (s *Rest) makeAutocertManager() *autocert.Manager {
 // makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
 func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	cfg := makeTLSConfig()
+	cfg := s.makeTLSConfig()
 	cfg.GetCertificate = m.GetCertificate
 	server.TLSConfig = cfg
 	return server
@@ -95,7 +95,7 @@ func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocer
 // makeHTTPSServer makes https server for static mode
 func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
 	server := s.makeHTTPServer(port, router)
-	server.TLSConfig = makeTLSConfig()
+	server.TLSConfig = s.makeTLSConfig()
 	return server
 }
 
@@ -109,7 +109,7 @@ func (s *Rest) getRemarkHost() string {
 	return u.Hostname()
 }
 
-func makeTLSConfig() *tls.Config {
+func (s *Rest) makeTLSConfig() *tls.Config {
 	return &tls.Config{
 		PreferServerCipherSuites: true,
 		CipherSuites: []uint16{

backend/app/rest/api/stream.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// Streamer creates endless stream of \n separated json records send to remote client
+type Streamer struct {
+	TimeOut     time.Duration
+	Refresh     time.Duration
+	MaxActive   int32
+	activeCount int32
+}
+
+type steamEventFn func() (event string, data []byte, upd bool, err error)
+
+type steamEventResp struct {
+	data  []byte
+	event string
+	err   error
+}
+
+// Activate starts blocking function streaming update created by eventFn to ResponseWriter
+// canceled on context or inactivity timeout
+// note: eventFn is a closure needed to allow state management inside eventFn
+func (s *Streamer) Activate(ctx context.Context, eventFn func() steamEventFn, w io.Writer) error {
+	updCh := s.eventsCh(ctx, eventFn())
+
+	count := atomic.AddInt32(&s.activeCount, 1)
+	defer atomic.AddInt32(&s.activeCount, -1)
+	if count > s.MaxActive {
+		return errors.New("too many streams")
+	}
+
+	if ww, ok := w.(http.ResponseWriter); ok {
+		ww.Header().Set("Content-Type", "text/event-stream")
+		ww.Header().Set("Connection", "keep-alive")
+		ww.Header().Set("Cache-Control", "no-cache")
+	}
+
+	for {
+		select {
+		case <-ctx.Done(): // request closed by remote client
+			log.Printf("[DEBUG] stream closed by remote client, %s", ctx.Err())
+			return nil
+		case <-time.After(s.TimeOut): // request closed by timeout
+			log.Printf("[DEBUG] stream closed due to timeout")
+			return nil
+		case resp, ok := <-updCh: // new update
+			if !ok { // closed updCh
+				return nil
+			}
+			if resp.err != nil {
+				return resp.err
+			}
+
+			// make server-sent event record
+			// see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+			if _, e := fmt.Fprintf(w, "event: %s\ndata: %s\n", resp.event, string(resp.data)); e != nil {
+				return errors.Wrap(e, "send to stream failed")
+			}
+			if fw, okFlush := w.(http.Flusher); okFlush {
+				fw.Flush()
+			}
+		}
+	}
+}
+
+// populate updates to chan, break on context close
+func (s *Streamer) eventsCh(ctx context.Context, fn steamEventFn) <-chan steamEventResp {
+	ch := make(chan steamEventResp)
+	go func() {
+		tick := time.NewTicker(s.Refresh)
+		defer func() {
+			close(ch)
+			tick.Stop()
+		}()
+		for {
+			select {
+			case <-ctx.Done(): // request closed by remote client
+				return
+			case <-tick.C:
+				event, resp, upd, err := fn()
+				if err != nil {
+					ch <- steamEventResp{event: event, data: nil, err: errors.Wrap(err, "can't get stream data")}
+					return
+				}
+				if upd {
+					ch <- steamEventResp{event: event, data: resp, err: nil}
+				}
+			}
+		}
+	}()
+	return ch
+}

backend/app/rest/api/stream_test.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStream_Timeout(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 || n > 10 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	err := s.Activate(context.Background(), eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}
+
+func TestStream_Cancel(t *testing.T) {
+	s := Streamer{
+		Refresh:   10 * time.Millisecond,
+		TimeOut:   100 * time.Millisecond,
+		MaxActive: 10,
+	}
+
+	eventFn := func() steamEventFn {
+		n := 0
+		return func() (event string, data []byte, upd bool, err error) {
+			n++
+			if n%2 == 0 {
+				return "test", nil, false, nil
+			}
+			return "test", []byte(fmt.Sprintf("some data %d\n", n)), true, nil
+		}
+	}
+
+	buf := bytes.Buffer{}
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err := s.Activate(ctx, eventFn, &buf)
+	assert.NoError(t, err)
+	assert.Equal(t, "event: test\ndata: some data 1\n\nevent: test\ndata: some data 3\n\nevent: test\ndata: some data 5\n\nevent: test\ndata: some data 7\n\nevent: test\ndata: some data 9\n\n", buf.String())
+}

backend/app/rest/httperrors_test.go

@@ -50,11 +50,13 @@ func TestErrorDetailsMsg(t *testing.T) {
 func TestErrorDetailsMsgWithUser(t *testing.T) {
 	callerFn := func() {
 		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
+		require.NoError(t, err)
 		req.RemoteAddr = "127.0.0.1:1234"
 		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
 		require.Nil(t, err)
 		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456", 34567)
-		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
+		assert.Equal(t, "error details 123456 - error 500 - 500 (34567) - test/id - 127.0.0.1 - https://example." +
+			"com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:61 rest.TestErrorDetailsMsgWithUser]", msg)
 	}
 	callerFn()
 }

backend/app/rest/proxy/image.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
@@ -40,76 +39,75 @@ func (p Image) Convert(commentHTML string) string {
 	return p.replace(commentHTML, imgs)
 }
 
-// Routes returns router group to respond to proxied request
+// Handler returns http handler respond to proxied request
-func (p Image) Routes() chi.Router {
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
-	router := chi.NewRouter()
+
 	if !p.Enabled {
-		return router
+		http.Error(w, "proxy disabled", http.StatusNotImplemented)
+		return
 	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-			return
-		}
 
-		timeout := 60 * time.Second // default
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if p.Timeout > 0 {
+	if err != nil {
-			timeout = p.Timeout
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-		}
+		return
+	}
 
-		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	timeout := 60 * time.Second // default
-		defer cancel()
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
 
-		client := http.Client{Timeout: 30 * time.Second}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		var resp *http.Response
+	defer cancel()
-		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-			var e error
-			req, e := http.NewRequest("GET", string(src), nil)
-			if e != nil {
-				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-			}
-			resp, e = client.Do(req.WithContext(ctx))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
 
-		if resp.StatusCode != http.StatusOK {
+	client := http.Client{Timeout: 30 * time.Second}
-			w.WriteHeader(resp.StatusCode)
+	var resp *http.Response
-			return
+	err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-		}
+		var e error
-
+		req, e := http.NewRequest("GET", string(src), nil)
-		for k, v := range resp.Header {
+		if e != nil {
-			if strings.EqualFold(k, "Content-Type") {
+			return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
 		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
 	})
-	return router
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
+		return
+	}
+	defer func() {
+		if e := resp.Body.Close(); e != nil {
+			log.Printf("[WARN] can't close body, %s", e)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		w.WriteHeader(resp.StatusCode)
+		return
+	}
+
+	for k, v := range resp.Header {
+		if strings.EqualFold(k, "Content-Type") {
+			w.Header().Set(k, v[0])
+		}
+		if strings.EqualFold(k, "Content-Length") {
+			w.Header().Set(k, v[0])
+		}
+	}
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if _, e := io.Copy(w, resp.Body); e != nil {
+		log.Printf("[WARN] can't copy image stream, %s", e)
+	}
 }
 
 // extract gets all non-https images and return list of src

backend/app/rest/proxy/image_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -48,9 +49,11 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{Enabled: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
-		assert.Nil(t, err, "err in #%d", i)
+			res, err := img.extract(tt.inp)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+			assert.Nil(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
@@ -63,12 +66,11 @@ func TestPicture_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
@@ -92,12 +94,11 @@ func TestImage_Routes(t *testing.T) {
 
 func TestImage_RoutesTimedOut(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)

backend/app/rest/user.go

@@ -36,7 +36,15 @@ func GetUserInfo(r *http.Request) (user store.User, err error) {
 		Verified: u.BoolAttr("verified"),
 		Blocked:  u.BoolAttr("blocked"),
 	}, nil
+}
 
+// GetUserOrEmpty attempts to get user info from request and returns empty object if failed
+func GetUserOrEmpty(r *http.Request) store.User {
+	user, err := GetUserInfo(r)
+	if err != nil {
+		return store.User{}
+	}
+	return user
 }
 
 // SetUserInfo sets user into request context

backend/app/rpc/client.go

@@ -0,0 +1,74 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+)
+
+// Client implements remote engine and delegates all calls to remote http server
+type Client struct {
+	API        string
+	Client     http.Client
+	AuthUser   string
+	AuthPasswd string
+
+	id uint64
+}
+
+// Call remote server with given method and arguments
+func (r *Client) Call(method string, args ...interface{}) (*Response, error) {
+
+	var b []byte
+	var err error
+
+	switch {
+	case len(args) == 0:
+		b, err = json.Marshal(Request{Method: method, ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	case len(args) == 1 && reflect.TypeOf(args[0]).Kind() == reflect.Struct:
+		b, err = json.Marshal(Request{Method: method, Params: args[0], ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	default:
+		b, err = json.Marshal(Request{Method: method, Params: args, ID: atomic.AddUint64(&r.id, 1)})
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshaling failed for %s", method)
+		}
+	}
+
+	req, err := http.NewRequest("POST", r.API, bytes.NewBuffer(b))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to make request for %s", method)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	if r.AuthUser != "" && r.AuthPasswd != "" {
+		req.SetBasicAuth(r.AuthUser, r.AuthPasswd)
+	}
+	resp, err := r.Client.Do(req)
+	if err != nil {
+		return nil, errors.Wrapf(err, "remote call failed for %s", method)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, errors.Errorf("bad status %s for %s", resp.Status, method)
+	}
+
+	cr := Response{}
+	if err = json.NewDecoder(resp.Body).Decode(&cr); err != nil {
+		return nil, errors.Wrapf(err, "failed to decode response for %s", method)
+	}
+
+	if cr.Error != "" {
+		return nil, errors.New(cr.Error)
+	}
+	return &cr, nil
+}

backend/app/rpc/client_test.go

@@ -0,0 +1,97 @@
+package rpc
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClient_Call(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	resp, err := c.Call("test", 123, "abc")
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithObject(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":{"F1":123,"F2":"abc","F3":"2019-06-09T23:03:55Z"},"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	obj := struct {
+		F1 int
+		F2 string
+		F3 time.Time
+	}{
+		F1: 123,
+		F2: "abc",
+		F3: time.Date(2019, 6, 9, 23, 3, 55, 0, time.UTC),
+	}
+
+	resp, err := c.Call("test", obj)
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallWithNoParams(t *testing.T) {
+	ts := testServer(t, `{"method":"test","id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	resp, err := c.Call("test")
+	assert.NoError(t, err)
+	res := ""
+	err = json.Unmarshal(*resp.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestClient_CallError(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"error":"some error"}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	_, err := c.Call("test", 123, "abc")
+	assert.EqualError(t, err, "some error")
+}
+
+func TestClient_CallBadResponse(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345 invalid}`)
+	defer ts.Close()
+	c := Client{API: ts.URL, Client: http.Client{}}
+	_, err := c.Call("test", 123, "abc")
+	assert.NotNil(t, err)
+}
+
+func TestClient_CallBadRemote(t *testing.T) {
+	ts := testServer(t, `{"method":"test","params":[123,"abc"],"id":1}`, `{"result":"12345"}`)
+	defer ts.Close()
+	c := Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}
+	_, err := c.Call("test", 123)
+	assert.NotNil(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/rpc/remote.go

@@ -0,0 +1,23 @@
+// Package rpc implements client ans server for RPC-like communication with remote storage.
+// The protocol is somewhat simplified version of json-rpc with a single POST call sending
+// Request json (method name and the list of parameters) and receiving back json Response with "result" json
+// and error string
+package rpc
+
+import (
+	"encoding/json"
+)
+
+// Request encloses method name and all params
+type Request struct {
+	Method string      `json:"method"`
+	Params interface{} `json:"params,omitempty"`
+	ID     uint64      `json:"id"`
+}
+
+// Response encloses result and error received from remote server
+type Response struct {
+	Result *json.RawMessage `json:"result,omitempty"`
+	Error  string           `json:"error,omitempty"`
+	ID     uint64           `json:"id"`
+}

backend/app/rpc/server.go

@@ -0,0 +1,179 @@
+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/didip/tollbooth"
+	"github.com/didip/tollbooth_chi"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/render"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/logger"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/rest"
+)
+
+// Server is json-rpc server with an optional basic auth
+type Server struct {
+	API        string
+	AuthUser   string
+	AuthPasswd string
+	Version    string
+	AppName    string
+
+	funcs struct {
+		m    map[string]ServerFn
+		once sync.Once
+	}
+
+	httpServer struct {
+		*http.Server
+		sync.Mutex
+	}
+}
+
+// Encoder is a function to encode call's result to Response
+type Encoder func(id uint64, resp interface{}, e error) (Response, error)
+
+// ServerFn handler registered for each method with Add
+// Implementations provided by consumer and define response logic.
+type ServerFn func(id uint64, params json.RawMessage) Response
+
+// Run http server on given port
+func (s *Server) Run(port int) error {
+	if s.AuthUser == "" || s.AuthPasswd == "" {
+		log.Print("[WARN] extension server runs without auth")
+	}
+	if s.funcs.m == nil && len(s.funcs.m) == 0 {
+		return errors.Errorf("nothing mapped for dispatch, Add has to be called prior to Run")
+	}
+
+	router := chi.NewRouter()
+	router.Use(middleware.Throttle(1000), middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(R.AppInfo(s.AppName, "umputun", s.Version), R.Ping)
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler
+	router.Use(middleware.Timeout(5 * time.Second))
+	router.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(1000, nil)), middleware.NoCache)
+	router.Use(s.basicAuth)
+
+	router.Post(s.API, s.handler)
+
+	s.httpServer.Lock()
+	s.httpServer.Server = &http.Server{
+		Addr:              fmt.Sprintf(":%d", port),
+		Handler:           router,
+		ReadHeaderTimeout: 5 * time.Second,
+		WriteTimeout:      10 * time.Second,
+		IdleTimeout:       30 * time.Second,
+	}
+	s.httpServer.Unlock()
+
+	log.Printf("[INFO] listen on %d", port)
+	return s.httpServer.ListenAndServe()
+}
+
+// EncodeResponse convert anything to Response
+func (s *Server) EncodeResponse(id uint64, resp interface{}, e error) (Response, error) {
+	v, err := json.Marshal(&resp)
+	if err != nil {
+		return Response{}, err
+	}
+	if e != nil {
+		return Response{ID: id, Result: nil, Error: e.Error()}, nil
+	}
+	raw := json.RawMessage{}
+	if err = raw.UnmarshalJSON(v); err != nil {
+		return Response{}, err
+	}
+	return Response{ID: id, Result: &raw}, nil
+}
+
+// Shutdown http server
+func (s *Server) Shutdown() error {
+	s.httpServer.Lock()
+	defer s.httpServer.Unlock()
+	if s.httpServer.Server == nil {
+		return errors.Errorf("http server is not running")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return s.httpServer.Shutdown(ctx)
+}
+
+// Add method handler
+func (s *Server) Add(method string, fn ServerFn) {
+	s.httpServer.Lock()
+	defer s.httpServer.Unlock()
+	if s.httpServer.Server != nil {
+		log.Printf("[WARN] ignored method %s, can't be added to activated server", method)
+		return
+	}
+
+	s.funcs.once.Do(func() {
+		s.funcs.m = map[string]ServerFn{}
+	})
+
+	s.funcs.m[method] = fn
+	log.Printf("[INFO] add handler for %s", method)
+}
+
+// HandlersGroup alias for map of handlers
+type HandlersGroup map[string]ServerFn
+
+// Group of handlers with common prefix
+func (s *Server) Group(prefix string, m HandlersGroup) {
+	for k, v := range m {
+		s.Add(prefix+"."+k, v)
+	}
+}
+
+func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
+	req := struct {
+		ID     uint64           `json:"id"`
+		Method string           `json:"method"`
+		Params *json.RawMessage `json:"params"`
+	}{}
+
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, req.Method, 0)
+		return
+	}
+	fn, ok := s.funcs.m[req.Method]
+	if !ok {
+		rest.SendErrorJSON(w, r, http.StatusNotImplemented, errors.New("unsupported method"), req.Method, 0)
+		return
+	}
+
+	params := json.RawMessage{}
+	if req.Params != nil {
+		params = *req.Params
+	}
+
+	render.JSON(w, r, fn(req.ID, params))
+}
+
+func (s *Server) basicAuth(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if s.AuthUser == "" || s.AuthPasswd == "" {
+			h.ServeHTTP(w, r)
+			return
+		}
+
+		user, pass, ok := r.BasicAuth()
+		if user != s.AuthUser || pass != s.AuthPasswd || !ok {
+			w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+		h.ServeHTTP(w, r)
+	})
+}

backend/app/rpc/server_test.go

@@ -0,0 +1,245 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServerPrimitiveTypes(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+
+	type respData struct {
+		Res1 string
+		Res2 bool
+	}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	// check with direct http call
+	clientReq := Request{Method: "test", Params: []interface{}{"blah", 42, true}, ID: 123}
+	b := bytes.Buffer{}
+	require.NoError(t, json.NewEncoder(&b).Encode(clientReq))
+	resp, err := http.Post("http://127.0.0.1:9091/v1/cmd", "application/json", &b)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	data, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, `{"result":{"Res1":"res blah","Res2":true},"id":123}`+"\n", string(data))
+
+	// check with client call
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	r, err := c.Call("test", "blah", 42, true)
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+
+	res := respData{}
+	err = json.Unmarshal(*r.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+	assert.Equal(t, uint64(1), r.ID)
+}
+
+func TestServerWithObject(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+
+	type respData struct {
+		Res1 string
+		Res2 bool
+	}
+
+	type reqData struct {
+		Time time.Time
+		F1   string
+		F2   time.Duration
+	}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		arg := reqData{}
+		if err := json.Unmarshal(params, &arg); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", arg)
+
+		r, err := s.EncodeResponse(id, respData{"res blah", true}, nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	r, err := c.Call("test", reqData{Time: time.Now(), F1: "sawert", F2: time.Minute})
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+
+	res := respData{}
+	err = json.Unmarshal(*r.Result, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, respData{Res1: "res blah", Res2: true}, res)
+}
+
+func TestServerMethodNotImplemented(t *testing.T) {
+	s := Server{}
+	ts := httptest.NewServer(http.HandlerFunc(s.handler))
+	defer ts.Close()
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+
+	r := Request{Method: "blah"}
+	buf := bytes.Buffer{}
+	assert.NoError(t, json.NewEncoder(&buf).Encode(r))
+	resp, err := http.Post(ts.URL, "application/json", &buf)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNotImplemented, resp.StatusCode)
+
+	assert.EqualError(t, s.Shutdown(), "http server is not running")
+}
+
+func TestServerWithAuth(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, "res blah", nil)
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	time.Sleep(10 * time.Millisecond)
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+	r, err := c.Call("test", "blah", 42, true)
+	assert.NoError(t, err)
+	assert.Equal(t, "", r.Error)
+	val := ""
+	err = json.Unmarshal(*r.Result, &val)
+	assert.NoError(t, err)
+	assert.Equal(t, "res blah", val)
+
+	c = Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err = c.Call("test", "blah", 42, true)
+	assert.EqualError(t, err, "bad status 401 Unauthorized for test")
+}
+
+func TestServerErrReturn(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+
+	s.Add("test", func(id uint64, params json.RawMessage) Response {
+		args := []interface{}{}
+		if err := json.Unmarshal(params, &args); err != nil {
+			return Response{Error: err.Error()}
+		}
+		t.Logf("%+v", args)
+
+		assert.Equal(t, 3, len(args))
+		assert.Equal(t, "blah", args[0].(string))
+		assert.Equal(t, 42., args[1].(float64))
+		assert.Equal(t, true, args[2].(bool))
+
+		r, err := s.EncodeResponse(id, "res blah", errors.New("some error"))
+		assert.NoError(t, err)
+		return r
+	})
+
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}, AuthUser: "user", AuthPasswd: "passwd"}
+	_, err := c.Call("test", "blah", 42, true)
+	assert.EqualError(t, err, "some error")
+}
+
+func TestServerGroup(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+	s.Group("pre", HandlersGroup{
+		"fn1": func(id uint64, params json.RawMessage) Response {
+			return Response{}
+		},
+		"fn2": func(id uint64, params json.RawMessage) Response {
+			return Response{}
+		},
+	})
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err := c.Call("fn1")
+	assert.EqualError(t, err, "bad status 501 Not Implemented for fn1")
+
+	_, err = c.Call("pre.fn1")
+	assert.NoError(t, err)
+	_, err = c.Call("pre.fn2")
+	assert.NoError(t, err)
+}
+
+func TestServerAddLate(t *testing.T) {
+	s := Server{API: "/v1/cmd"}
+	s.Add("fn1", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+	go func() { _ = s.Run(9091) }()
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+	time.Sleep(10 * time.Millisecond)
+
+	// too late, ignored after run
+	s.Add("fn2", func(id uint64, params json.RawMessage) Response {
+		return Response{}
+	})
+
+	c := Client{API: "http://127.0.0.1:9091/v1/cmd", Client: http.Client{}}
+	_, err := c.Call("fn1")
+	assert.NoError(t, err)
+	_, err = c.Call("fn2")
+	assert.EqualError(t, err, "bad status 501 Not Implemented for fn2")
+}
+
+func TestServerNoHandlers(t *testing.T) {
+	s := Server{API: "/v1/cmd", AuthUser: "user", AuthPasswd: "passwd"}
+	assert.EqualError(t, s.Run(9091), "nothing mapped for dispatch, Add has to be called prior to Run")
+}

backend/app/store/admin/admin.go

@@ -10,25 +10,17 @@ import (
 // Store defines interface returning admins info for given site
 type Store interface {
 	Key() (key string, err error)
-	Admins(siteID string) (ids []string)
+	Admins(siteID string) (ids []string, err error)
-	Email(siteID string) (email string)
+	Email(siteID string) (email string, err error)
 }
 
-// StaticStore implements keys.Store with a single, predefined key
+// StaticStore implements keys.Store with a single set of admins and email for all sites
 type StaticStore struct {
 	admins []string
 	email  string
 	key    string
 }
 
-// Key returns static key for all sites, allows empty site
-func (s *StaticStore) Key() (key string, err error) {
-	if s.key == "" {
-		return "", errors.New("empty key for static key store")
-	}
-	return s.key, nil
-}
-
 // NewStaticStore makes StaticStore instance with given key
 func NewStaticStore(key string, admins []string, email string) *StaticStore {
 	log.Printf("[DEBUG] admin users %+v, email %s", admins, email)
@@ -40,12 +32,20 @@ func NewStaticKeyStore(key string) *StaticStore {
 	return &StaticStore{key: key, admins: []string{}, email: ""}
 }
 
+// Key returns static key, same for all sites
+func (s *StaticStore) Key() (key string, err error) {
+	if s.key == "" {
+		return "", errors.New("empty key for static key store")
+	}
+	return s.key, nil
+}
+
 // Admins returns static list of admin's ids, the same for all sites
-func (s *StaticStore) Admins(string) (ids []string) {
+func (s *StaticStore) Admins(string) (ids []string, err error) {
-	return s.admins
+	return s.admins, nil
 }
 
 // Email gets static email address
-func (s *StaticStore) Email(string) (email string) {
+func (s *StaticStore) Email(string) (email string, err error) {
-	return s.email
+	return s.email, nil
 }

backend/app/store/admin/admin_test.go

@@ -3,10 +3,7 @@ package admin
 import (
 	"testing"
 
-	"github.com/globalsign/mgo"
-	"github.com/go-pkgz/mongo"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestStaticStore_Get(t *testing.T) {
@@ -16,48 +13,11 @@ func TestStaticStore_Get(t *testing.T) {
 	assert.NoError(t, err, "valid store")
 	assert.Equal(t, "key123", k, "valid site")
 
-	a := ks.Admins("any")
+	a, err := ks.Admins("any")
+	assert.NoError(t, err)
 	assert.Equal(t, []string{"123", "xyz"}, a)
 
-	email := ks.Email("blah")
+	email, err := ks.Email("blah")
+	assert.NoError(t, err)
 	assert.Equal(t, "aa@example.com", email)
 }
-
-func TestMongoStore_Get(t *testing.T) {
-	conn, err := mongo.MakeTestConnection(t)
-	require.NoError(t, err)
-	var ms Store = NewMongoStore(conn, "secret")
-
-	recs := []mongoRec{
-		{"site1", []string{"i11", "i12"}, "e1"},
-		{"site2", []string{"i21", "i22"}, "e2"},
-	}
-	err = conn.WithCollection(func(coll *mgo.Collection) error {
-		if e1 := coll.Insert(recs[0]); e1 != nil {
-			return e1
-		}
-		return coll.Insert(recs[1])
-	})
-	require.NoError(t, err)
-
-	admins := ms.Admins("site1")
-	assert.Equal(t, []string{"i11", "i12"}, admins)
-	email := ms.Email("site1")
-	assert.Equal(t, "e1", email)
-	key, err := ms.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("site2")
-	assert.Equal(t, []string{"i21", "i22"}, admins)
-	email = ms.Email("site2")
-	assert.Equal(t, "e2", email)
-	key, err = ms.Key()
-	assert.NoError(t, err)
-	assert.Equal(t, "secret", key)
-
-	admins = ms.Admins("no-site-in-db")
-	assert.Equal(t, []string{}, admins)
-	email = ms.Email("no-site-in-db")
-	assert.Equal(t, "", email)
-}

backend/app/store/admin/mongo.go

@@ -1,56 +0,0 @@
-package admin
-
-import (
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	log "github.com/go-pkgz/lgr"
-
-	"github.com/go-pkgz/mongo"
-)
-
-// MongoStore implements admin.Store with mongo backend
-type MongoStore struct {
-	connection *mongo.Connection
-	key        string
-}
-
-type mongoRec struct {
-	SiteID string   `bson:"site"`
-	IDs    []string `bson:"admin_ids"`
-	Email  string   `bson:"admin_email"`
-}
-
-// NewMongoStore makes admin Store for mongo's connection
-func NewMongoStore(conn *mongo.Connection, key string) *MongoStore {
-	log.Printf("[DEBUG] make mongo admin store with %+v", conn)
-	return &MongoStore{connection: conn, key: key}
-}
-
-// Key executes find by siteID and returns substructure with secret key
-func (m *MongoStore) Key() (key string, err error) {
-	return m.key, nil
-}
-
-// Admins executes find by siteID and returns admins ids
-func (m *MongoStore) Admins(siteID string) (ids []string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return []string{}
-	}
-	return resp.IDs
-}
-
-// Email executes find by siteID and returns admin's email
-func (m *MongoStore) Email(siteID string) (email string) {
-	resp := mongoRec{}
-	err := m.connection.WithCollection(func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID}).One(&resp)
-	})
-	if err != nil {
-		return ""
-	}
-	return resp.Email
-}

backend/app/store/admin/remote.go

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"encoding/json"
+
+	"github.com/umputun/remark/backend/app/rpc"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	rpc.Client
+}
+
+// Key returns the key, same for all sites
+func (r *RPC) Key() (key string, err error) {
+	resp, err := r.Call("admin.key")
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &key)
+	return key, err
+}
+
+// Admins returns list of admin's ids for given site
+func (r *RPC) Admins(siteID string) (ids []string, err error) {
+	resp, err := r.Call("admin.admins", siteID)
+	if err != nil {
+		return []string{}, err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &ids); err != nil {
+		return []string{}, err
+	}
+	return ids, nil
+}
+
+// Email gets email address for given site
+func (r *RPC) Email(siteID string) (email string, err error) {
+	resp, err := r.Call("admin.email", siteID)
+	if err != nil {
+		return "", err
+	}
+
+	if err = json.Unmarshal(*resp.Result, &email); err != nil {
+		return "", err
+	}
+	return email, nil
+}

backend/app/store/admin/remote_test.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package admin
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/rpc"
+)
+
+func TestRemote_Key(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.key","id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Key()
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Admins(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.admins","params":["site-1"],"id":1}`,
+		`{"result":["id1","id2"],"id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Admins("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Email(t *testing.T) {
+	ts := testServer(t, `{"method":"admin.email","params":["site-1"],"id":1}`,
+		`{"result":"bbb@example.com","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var a Store = &c
+	_ = a
+
+	res, err := c.Email("site-1")
+	assert.NoError(t, err)
+	assert.Equal(t, "bbb@example.com", res)
+	t.Logf("%v %T", res, res)
+}
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/store/comment.go

@@ -3,6 +3,7 @@ package store
 import (
 	"html/template"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/microcosm-cc/bluemonday"
@@ -66,6 +67,7 @@ const (
 
 // Maximum length for URL text shortening.
 const shortURLLen = 48
+const snippetLen = 200
 
 // PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
 // autogen fields and reset everything users not supposed to provide
@@ -107,3 +109,24 @@ func (c *Comment) Sanitize() {
 	c.User.Name = template.HTMLEscapeString(c.User.Name)
 	c.User.Picture = p.Sanitize(c.User.Picture)
 }
+
+// Snippet from comment's text
+func (c *Comment) Snippet(limit int) string {
+	if limit <= 0 {
+		limit = snippetLen
+	}
+	cleanText := strings.Replace(c.Text, "\n", " ", -1)
+	size := len([]rune(cleanText))
+	if size < limit {
+		return cleanText
+	}
+	snippet := []rune(cleanText)[:size]
+	// go back in snippet and found the first space
+	for i := len(snippet) - 1; i >= 0; i-- {
+		if snippet[i] == ' ' {
+			snippet = snippet[:i]
+			break
+		}
+	}
+	return string(snippet) + " ..."
+}

backend/app/store/comment_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -34,6 +35,14 @@ func TestComment_Sanitize(t *testing.T) {
 				User: User{ID: "id", Name: "xyz"},
 			},
 		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
+		{
+			inp: Comment{Text: "blah & & 123 — —"},
+			out: Comment{Text: `blah & & 123 — —`},
+		},
 	}
 
 	for n, tt := range tbl {
@@ -120,3 +129,25 @@ func TestComment_SetDeletedHard(t *testing.T) {
 	assert.False(t, comment.Pin)
 	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
 }
+
+func TestComment_Snippet(t *testing.T) {
+	tbl := []struct {
+		limit int
+		inp   string
+		out   string
+	}{
+		{0, "", ""},
+		{-1, "test\nblah", "test blah"},
+		{5, "test\nblah", "test ..."},
+		{5, "xyz12345 xxx", "xyz12345 ..."},
+		{10, "xyz12345 xxx\ntest 123456", "xyz12345 xxx test ..."},
+	}
+
+	for i, tt := range tbl {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			c := Comment{Text: tt.inp}
+			out := c.Snippet(tt.limit)
+			assert.Equal(t, tt.out, out)
+		})
+	}
+}

backend/app/store/engine/bolt.go

@@ -0,0 +1,866 @@
+package engine
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
+// there are 5 types of top-level buckets:
+//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
+//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
+//    value is not full comment but a reference combined from post-url+commentID
+//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
+//    is a nested bucket named userID with kv as ts:reference
+//  - blocking info sits in "block" bucket. Key is userID, value - ts
+//  - counts per post to keep number of comments. Key is post url, value - count
+//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
+type BoltDB struct {
+	dbs map[string]*bolt.DB
+}
+
+const (
+	// top level buckets
+	postsBucketName    = "posts"
+	lastBucketName     = "last"
+	userBucketName     = "users"
+	blocksBucketName   = "block"
+	infoBucketName     = "info"
+	readonlyBucketName = "readonly"
+	verifiedBucketName = "verified"
+
+	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
+)
+
+// BoltSite defines single site param
+type BoltSite struct {
+	FileName string // full path to boltdb
+	SiteID   string // ID of given site
+}
+
+// NewBoltDB makes persistent boltdb-based store. For each site new boltdb file created
+func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
+	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
+	result := BoltDB{dbs: make(map[string]*bolt.DB)}
+	for _, site := range sites {
+		db, err := bolt.Open(site.FileName, 0600, &options)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
+		}
+
+		// make top-level buckets
+		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName, infoBucketName,
+			readonlyBucketName, verifiedBucketName}
+		err = db.Update(func(tx *bolt.Tx) error {
+			for _, bktName := range topBuckets {
+				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+				}
+			}
+			return nil
+		})
+
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create top level bucket)")
+		}
+
+		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
+	}
+	return &result, nil
+}
+
+// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
+func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
+
+	bdb, err := b.db(comment.Locator.SiteID)
+	if err != nil {
+		return "", err
+	}
+
+	if b.checkFlag(FlagRequest{Locator: comment.Locator, Flag: ReadOnly}) {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	err = bdb.Update(func(tx *bolt.Tx) (err error) {
+		var postBkt, lastBkt, userBkt *bolt.Bucket
+
+		if postBkt, err = b.makePostBucket(tx, comment.Locator.URL); err != nil {
+			return err
+		}
+		// check if key already in store, reject doubles
+		if postBkt.Get([]byte(comment.ID)) != nil {
+			return errors.Errorf("key %s already in store", comment.ID)
+		}
+
+		// serialize comment to json []byte for bolt and save
+		if err = b.save(postBkt, comment.ID, comment); err != nil {
+			return errors.Wrapf(err, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
+		}
+
+		ref := b.makeRef(comment) // reference combines url and comment id
+
+		// add reference to comment to "last" bucket
+		lastBkt = tx.Bucket([]byte(lastBucketName))
+		commentTs := []byte(comment.Timestamp.Format(tsNano))
+		if err = lastBkt.Put(commentTs, ref); err != nil {
+			return errors.Wrapf(err, "can't put reference %s to %s", ref, lastBucketName)
+		}
+
+		// add reference to commentID to "users" bucket
+		if userBkt, err = b.getUserBucket(tx, comment.User.ID); err != nil {
+			return errors.Wrapf(err, "can't get bucket %s", comment.User.ID)
+		}
+		// put into individual user's bucket with ts as a key
+		if err = userBkt.Put(commentTs, ref); err != nil {
+			return errors.Wrapf(err, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
+		}
+
+		// set info with the count for post url
+		if _, err = b.setInfo(tx, comment); err != nil {
+			return errors.Wrapf(err, "failed to set info for %s", comment.Locator)
+		}
+		return nil
+	})
+
+	return comment.ID, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (b *BoltDB) Get(req GetRequest) (comment store.Comment, err error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return comment, err
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		bucket, e := b.getPostBucket(tx, req.Locator.URL)
+		if e != nil {
+			return e
+		}
+		return b.load(bucket, req.CommentID, &comment)
+	})
+	return comment, err
+}
+
+// Find returns all comments for given request and sorts results
+func (b *BoltDB) Find(req FindRequest) (comments []store.Comment, err error) {
+	comments = []store.Comment{}
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find post comments, i.e. for site and url
+		err = bdb.View(func(tx *bolt.Tx) error {
+
+			bucket, e := b.getPostBucket(tx, req.Locator.URL)
+			if e != nil {
+				return e
+			}
+
+			return bucket.ForEach(func(k, v []byte) error {
+				comment := store.Comment{}
+				if e = json.Unmarshal(v, &comment); e != nil {
+					return errors.Wrap(e, "failed to unmarshal")
+				}
+				if req.Since.IsZero() || comment.Timestamp.After(req.Since) {
+					comments = append(comments, comment)
+				}
+				return nil
+			})
+		})
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		comments, err = b.lastComments(req.Locator.SiteID, req.Limit, req.Since)
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments, err = b.userComments(req.Locator.SiteID, req.UserID, req.Limit, req.Skip)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	return SortComments(comments, req.Sort), nil
+}
+
+// Flag sets and gets flag values
+func (b *BoltDB) Flag(req FlagRequest) (val bool, err error) {
+	if req.Update == FlagNonSet { // read flag value, no update requested
+		return b.checkFlag(req), nil
+	}
+
+	// write flag value
+	return b.setFlag(req)
+}
+
+// Update for locator.URL with mutable part of comment
+func (b *BoltDB) Update(comment store.Comment) error {
+
+	getReq := GetRequest{Locator: comment.Locator, CommentID: comment.ID}
+	if curComment, err := b.Get(getReq); err == nil {
+		// preserve immutable fields
+		comment.ParentID = curComment.ParentID
+		comment.Locator = curComment.Locator
+		comment.Timestamp = curComment.Timestamp
+		comment.User = curComment.User
+	}
+
+	bdb, err := b.db(comment.Locator.SiteID)
+	if err != nil {
+		return err
+	}
+
+	return bdb.Update(func(tx *bolt.Tx) error {
+		bucket, e := b.getPostBucket(tx, comment.Locator.URL)
+		if e != nil {
+			return e
+		}
+		return b.save(bucket, comment.ID, comment)
+	})
+}
+
+// Count returns number of comments for post or user
+func (b *BoltDB) Count(req FindRequest) (count int, err error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return 0, err
+	}
+
+	if req.Locator.URL != "" { // comment's count for post
+		err = bdb.View(func(tx *bolt.Tx) error {
+			var e error
+			count, e = b.count(tx, req.Locator.URL, 0)
+			return e
+		})
+		return count, err
+	}
+
+	if req.UserID != "" { // comment's count for user
+		err = bdb.View(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(userBucketName))
+			userIDBkt := usersBkt.Bucket([]byte(req.UserID))
+			if userIDBkt == nil {
+				return errors.Errorf("no comments for user %s in store for %s site", req.UserID, req.Locator.SiteID)
+			}
+			stats := userIDBkt.Stats()
+			count = stats.KeyN
+			return nil
+		})
+		return count, err
+	}
+
+	return 0, errors.Errorf("invalid count request %+v", req)
+}
+
+// Info get post(s) meta info
+func (b *BoltDB) Info(req InfoRequest) ([]store.PostInfo, error) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return []store.PostInfo{}, err
+	}
+
+	if req.Locator.URL != "" { // post info
+		info := store.PostInfo{}
+		err = bdb.View(func(tx *bolt.Tx) error {
+			infoBkt := tx.Bucket([]byte(infoBucketName))
+			if e := b.load(infoBkt, req.Locator.URL, &info); e != nil {
+				return errors.Wrapf(e, "can't load info for %s", req.Locator.URL)
+			}
+			return nil
+		})
+
+		// set read-only from age and manual bucket
+		readOnlyAge := req.ReadOnlyAge
+		info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+		if b.checkFlag(FlagRequest{Locator: req.Locator, Flag: ReadOnly}) {
+			info.ReadOnly = true
+		}
+		return []store.PostInfo{info}, err
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		list := []store.PostInfo{}
+		err = bdb.View(func(tx *bolt.Tx) error {
+			postsBkt := tx.Bucket([]byte(postsBucketName))
+
+			c := postsBkt.Cursor()
+			n := 0
+			for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
+				n++
+				if req.Skip > 0 && n <= req.Skip {
+					continue
+				}
+				postURL := string(k)
+				infoBkt := tx.Bucket([]byte(infoBucketName))
+				info := store.PostInfo{}
+				if e := b.load(infoBkt, postURL, &info); e != nil {
+					return errors.Wrapf(e, "can't load info for %s", postURL)
+				}
+				list = append(list, info)
+				if req.Limit > 0 && len(list) >= req.Limit {
+					break
+				}
+			}
+			return nil
+		})
+		return list, err
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (b *BoltDB) ListFlags(req FlagRequest) (res []interface{}, err error) {
+
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return nil, e
+	}
+
+	res = []interface{}{}
+	switch req.Flag {
+	case Verified:
+		err = bdb.View(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(verifiedBucketName))
+			_ = usersBkt.ForEach(func(k, _ []byte) error {
+				res = append(res, string(k))
+				return nil
+			})
+			return nil
+		})
+		return res, err
+	case Blocked:
+		err = bdb.View(func(tx *bolt.Tx) error {
+			bucket := tx.Bucket([]byte(blocksBucketName))
+			return bucket.ForEach(func(k []byte, v []byte) error {
+				ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+				if errParse != nil {
+					return errors.Wrap(errParse, "can't parse block ts")
+				}
+				if time.Now().Before(ts) {
+					// get user name from comment user section
+					userName := ""
+					findReq := FindRequest{Locator: store.Locator{SiteID: req.Locator.SiteID}, UserID: string(k), Limit: 1}
+					userComments, errUser := b.Find(findReq)
+					if errUser == nil && len(userComments) > 0 {
+						userName = userComments[0].User.Name
+					}
+					res = append(res, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
+				}
+				return nil
+			})
+		})
+		return res, err
+	}
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// Delete post(s) by id or by userID
+func (b *BoltDB) Delete(req DeleteRequest) error {
+
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return e
+	}
+
+	switch {
+	case req.Locator.URL != "" && req.CommentID != "":
+		return b.deleteComment(bdb, req.Locator, req.CommentID, req.DeleteMode)
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "":
+		return b.deleteUser(bdb, req.Locator.SiteID, req.UserID, req.DeleteMode)
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "":
+		return b.deleteAll(bdb, req.Locator.SiteID)
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+// Close boltdb store
+func (b *BoltDB) Close() error {
+	errs := new(multierror.Error)
+	for site, db := range b.dbs {
+		err := errors.Wrapf(db.Close(), "can't close site %s", site)
+		errs = multierror.Append(errs, err)
+	}
+	return errs.ErrorOrNil()
+}
+
+// Last returns up to max last comments for given siteID
+func (b *BoltDB) lastComments(siteID string, max int, since time.Time) (comments []store.Comment, err error) {
+
+	comments = []store.Comment{}
+
+	if max > lastLimit || max == 0 {
+		max = lastLimit
+	}
+
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return nil, err
+	}
+
+	err = bdb.View(func(tx *bolt.Tx) error {
+		lastBkt := tx.Bucket([]byte(lastBucketName))
+		c := lastBkt.Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+
+			if !since.IsZero() {
+				// stop if reached "since" ts
+				tsSince := []byte(since.Format(tsNano))
+				if bytes.Compare(k, tsSince) <= 0 {
+					break
+				}
+			}
+			url, commentID, e := b.parseRef(v)
+			if e != nil {
+				return e
+			}
+			postBkt, e := b.getPostBucket(tx, url)
+			if e != nil {
+				return e
+			}
+
+			comment := store.Comment{}
+			if e = b.load(postBkt, commentID, &comment); e != nil {
+				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
+				continue
+			}
+			if comment.Deleted {
+				continue
+			}
+			comments = append(comments, comment)
+			if len(comments) >= max {
+				break
+			}
+		}
+		return nil
+	})
+
+	return comments, err
+}
+
+// userComments extracts all comments for given site and given userID
+// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
+func (b *BoltDB) userComments(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+
+	comments = []store.Comment{}
+	commentRefs := []string{}
+
+	bdb, err := b.db(siteID)
+	if err != nil {
+		return nil, err
+	}
+
+	if limit == 0 || limit > userLimit {
+		limit = userLimit
+	}
+
+	// get list of references to comments
+	err = bdb.View(func(tx *bolt.Tx) error {
+		usersBkt := tx.Bucket([]byte(userBucketName))
+		userIDBkt := usersBkt.Bucket([]byte(userID))
+		if userIDBkt == nil {
+			return errors.Errorf("no comments for user %s in store", userID)
+		}
+
+		c := userIDBkt.Cursor()
+		skipComments := 0
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			if len(commentRefs) >= limit {
+				break
+			}
+			if skip > 0 && skipComments < skip {
+				skipComments++
+				continue
+			}
+			commentRefs = append(commentRefs, string(v))
+		}
+		return nil
+	})
+
+	if err != nil {
+		return comments, err
+	}
+
+	// retrieve comments for refs
+	for _, v := range commentRefs {
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
+		}
+		getReq := GetRequest{Locator: store.Locator{SiteID: siteID, URL: url}, CommentID: commentID}
+		if c, errRef := b.Get(getReq); errRef == nil {
+			comments = append(comments, c)
+		}
+	}
+
+	return comments, err
+}
+
+func (b *BoltDB) checkFlag(req FlagRequest) (val bool) {
+
+	bdb, err := b.db(req.Locator.SiteID)
+	if err != nil {
+		return false
+	}
+
+	key := req.Locator.URL
+	if req.UserID != "" {
+		key = req.UserID
+	}
+
+	if req.Flag == Blocked {
+		var blocked bool
+		_ = bdb.View(func(tx *bolt.Tx) error {
+			bucket := tx.Bucket([]byte(blocksBucketName))
+			v := bucket.Get([]byte(key))
+			if v == nil {
+				blocked = false
+				return nil
+			}
+
+			until, e := time.Parse(tsNano, string(v))
+			if e != nil {
+				blocked = false
+				return nil
+			}
+			blocked = time.Now().Before(until)
+			return nil
+		})
+		return blocked
+	}
+
+	_ = bdb.View(func(tx *bolt.Tx) error {
+		var bucket *bolt.Bucket
+		if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+			return err
+		}
+		val = bucket.Get([]byte(key)) != nil
+		return nil
+	})
+	return val
+}
+
+func (b *BoltDB) setFlag(req FlagRequest) (res bool, err error) {
+	bdb, e := b.db(req.Locator.SiteID)
+	if e != nil {
+		return false, e
+	}
+
+	key := req.Locator.URL
+	if req.UserID != "" {
+		key = req.UserID
+	}
+
+	err = bdb.Update(func(tx *bolt.Tx) error {
+		var bucket *bolt.Bucket
+		if bucket, err = b.flagBucket(tx, req.Flag); err != nil {
+			return err
+		}
+		switch req.Update {
+		case FlagTrue:
+			if req.Flag == Blocked {
+				val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
+				if req.TTL > 0 {
+					val = time.Now().Add(req.TTL).Format(tsNano)
+				}
+				if e = bucket.Put([]byte(key), []byte(val)); e != nil {
+					return errors.Wrapf(e, "failed to put blocked to %s", key)
+				}
+				res = true
+				return nil
+			}
+
+			if e = bucket.Put([]byte(key), []byte(time.Now().Format(tsNano))); e != nil {
+				return errors.Wrapf(e, "failed to set flag %s for %s", req.Flag, req.Locator.URL)
+			}
+			res = true
+			return nil
+		case FlagFalse:
+			if e = bucket.Delete([]byte(key)); e != nil {
+				return errors.Wrapf(e, "failed to clean flag %s for %s", req.Flag, req.Locator.URL)
+			}
+			res = false
+		}
+		return nil
+	})
+
+	return res, err
+}
+
+func (b *BoltDB) flagBucket(tx *bolt.Tx, flag Flag) (bkt *bolt.Bucket, err error) {
+	switch flag {
+	case ReadOnly:
+		bkt = tx.Bucket([]byte(readonlyBucketName))
+	case Blocked:
+		bkt = tx.Bucket([]byte(blocksBucketName))
+	case Verified:
+		bkt = tx.Bucket([]byte(verifiedBucketName))
+	default:
+		return nil, errors.Errorf("unsupported flag %v", flag)
+	}
+	return bkt, nil
+}
+
+func (b *BoltDB) deleteComment(bdb *bolt.DB, locator store.Locator, commentID string, mode store.DeleteMode) error {
+
+	return bdb.Update(func(tx *bolt.Tx) error {
+
+		postBkt, e := b.getPostBucket(tx, locator.URL)
+		if e != nil {
+			return e
+		}
+
+		comment := store.Comment{}
+		if e = b.load(postBkt, commentID, &comment); e != nil {
+			return errors.Wrapf(e, "can't load key %s from bucket %s", commentID, locator.URL)
+		}
+		// set deleted status and clear fields
+		comment.SetDeleted(mode)
+
+		if e = b.save(postBkt, commentID, comment); e != nil {
+			return errors.Wrapf(e, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
+		}
+
+		// delete from "last" bucket
+		lastBkt := tx.Bucket([]byte(lastBucketName))
+		if e = lastBkt.Delete([]byte(commentID)); e != nil {
+			return errors.Wrapf(e, "can't delete key %s from bucket %s", commentID, lastBucketName)
+		}
+
+		// decrement comments count for post url
+		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
+			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
+		}
+
+		return nil
+	})
+}
+
+// deleteAll removes all top-level buckets for given siteID
+func (b *BoltDB) deleteAll(bdb *bolt.DB, siteID string) error {
+
+	// delete all buckets except blocked users
+	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
+
+	// delete top-level buckets
+	err := bdb.Update(func(tx *bolt.Tx) error {
+		for _, bktName := range toDelete {
+
+			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
+				return errors.Wrapf(e, "failed to delete top level bucket %s", bktName)
+			}
+			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
+				return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
+			}
+		}
+		return nil
+	})
+
+	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
+}
+
+// deleteUser removes all comments for given user. Everything will be market as deleted
+// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
+func (b *BoltDB) deleteUser(bdb *bolt.DB, siteID string, userID string, mode store.DeleteMode) error {
+
+	// get list of all comments outside of transaction loop
+	posts, err := b.Info(InfoRequest{Locator: store.Locator{SiteID: siteID}})
+	if err != nil {
+		return err
+	}
+
+	type commentInfo struct {
+		locator   store.Locator
+		commentID string
+	}
+
+	// get list of commentID for all user's comment
+	comments := []commentInfo{}
+	for _, postInfo := range posts {
+		err = bdb.View(func(tx *bolt.Tx) error {
+			postsBkt := tx.Bucket([]byte(postsBucketName))
+			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
+			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
+				comment := store.Comment{}
+				if err = json.Unmarshal(commentVal, &comment); err != nil {
+					return errors.Wrap(err, "failed to unmarshal")
+				}
+				if comment.User.ID == userID {
+					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
+				}
+				return nil
+			})
+			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Printf("[DEBUG] comments for removal=%d", len(comments))
+
+	// delete collected comments
+	for _, ci := range comments {
+		if e := b.deleteComment(bdb, ci.locator, ci.commentID, mode); e != nil {
+			return errors.Wrapf(err, "failed to delete comment %+v", ci)
+		}
+	}
+
+	//  delete  user bucket in hard mode
+	if mode == store.HardDelete {
+		err = bdb.Update(func(tx *bolt.Tx) error {
+			usersBkt := tx.Bucket([]byte(userBucketName))
+			if usersBkt != nil {
+				if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
+					return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
+				}
+			}
+			return nil
+		})
+
+		if err != nil {
+			return errors.Wrap(err, "can't delete user meta")
+		}
+	}
+
+	if len(comments) == 0 {
+		return errors.Errorf("unknown user %s", userID)
+	}
+
+	return err
+}
+
+// getPostBucket return bucket with all comments for postURL
+func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+	postsBkt := tx.Bucket([]byte(postsBucketName))
+	if postsBkt == nil {
+		return nil, errors.Errorf("no bucket %s", postsBucketName)
+	}
+	res := postsBkt.Bucket([]byte(postURL))
+	if res == nil {
+		return nil, errors.Errorf("no bucket %s in store", postURL)
+	}
+	return res, nil
+}
+
+// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
+func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
+	postsBkt := tx.Bucket([]byte(postsBucketName))
+	if postsBkt == nil {
+		return nil, errors.Errorf("no bucket %s", postsBucketName)
+	}
+	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
+	if err != nil {
+		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
+	}
+	return res, nil
+}
+
+func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
+	usersBkt := tx.Bucket([]byte(userBucketName))
+	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
+	if e != nil {
+		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
+	}
+	return userIDBkt, nil
+}
+
+// save marshaled value to key for bucket. Should run in update tx
+func (b *BoltDB) save(bkt *bolt.Bucket, key string, value interface{}) (err error) {
+	if value == nil {
+		return errors.Errorf("can't save nil value for %s", key)
+	}
+	jdata, jerr := json.Marshal(value)
+	if jerr != nil {
+		return errors.Wrap(jerr, "can't marshal comment")
+	}
+	if err = bkt.Put([]byte(key), jdata); err != nil {
+		return errors.Wrapf(err, "failed to save key %s", key)
+	}
+	return nil
+}
+
+// load and unmarshal json value by key from bucket. Should run in view tx
+func (b *BoltDB) load(bkt *bolt.Bucket, key string, res interface{}) error {
+	value := bkt.Get([]byte(key))
+	if value == nil {
+		return errors.Errorf("no value for %s", key)
+	}
+
+	if err := json.Unmarshal(value, &res); err != nil {
+		return errors.Wrap(err, "failed to unmarshal")
+	}
+	return nil
+}
+
+// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
+// it uses separate counts bucket because boltdb Stat call is very slow
+func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
+
+	infoBkt := tx.Bucket([]byte(infoBucketName))
+
+	info := store.PostInfo{}
+	if err := b.load(infoBkt, postURL, &info); err != nil {
+		info = store.PostInfo{}
+	}
+	if val == 0 { // get current count, don't update
+		return info.Count, nil
+	}
+	info.Count += val
+
+	return info.Count, b.save(infoBkt, postURL, &info)
+}
+
+func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
+	infoBkt := tx.Bucket([]byte(infoBucketName))
+	info := store.PostInfo{}
+	if err := b.load(infoBkt, comment.Locator.URL, &info); err != nil {
+		info = store.PostInfo{
+			Count:   0,
+			URL:     comment.Locator.URL,
+			FirstTS: comment.Timestamp,
+			LastTS:  comment.Timestamp,
+		}
+	}
+	info.Count++
+	info.LastTS = comment.Timestamp
+	return info, b.save(infoBkt, comment.Locator.URL, &info)
+}
+
+func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
+	if res, ok := b.dbs[siteID]; ok {
+		return res, nil
+	}
+	return nil, errors.Errorf("site %q not found", siteID)
+}
+
+// makeRef creates reference combining url and comment id
+func (b *BoltDB) makeRef(comment store.Comment) []byte {
+	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
+}
+
+// parseRef gets parts of reference
+func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
+	elems := strings.Split(string(val), "!!")
+	if len(elems) != 2 {
+		return "", "", errors.Errorf("invalid reference value %s", string(val))
+	}
+	return elems[0], elems[1], nil
+}

backend/app/store/engine/bolt_accessor.go

@@ -1,540 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
-	dbs map[string]*bolt.DB
-}
-
-const (
-	// top level buckets
-	postsBucketName    = "posts"
-	lastBucketName     = "last"
-	userBucketName     = "users"
-	blocksBucketName   = "block"
-	infoBucketName     = "info"
-	readonlyBucketName = "readonly"
-	verifiedBucketName = "verified"
-
-	tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-)
-
-// BoltSite defines single site param
-type BoltSite struct {
-	FileName string // full path to boltdb
-	SiteID   string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
-	log.Printf("[INFO] bolt store for sites %+v, options %+v", sites, options)
-	result := BoltDB{dbs: make(map[string]*bolt.DB)}
-	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
-		}
-
-		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
-			infoBucketName, readonlyBucketName, verifiedBucketName}
-		err = db.Update(func(tx *bolt.Tx) error {
-			for _, bktName := range topBuckets {
-				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
-				}
-			}
-			return nil
-		})
-
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create top level bucket)")
-		}
-
-		result.dbs[site.SiteID] = db
-		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
-	}
-	return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
-	bdb, err := b.db(comment.Locator.SiteID)
-	if err != nil {
-		return "", err
-	}
-
-	if b.IsReadOnly(comment.Locator) {
-		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
-	}
-
-	err = bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
-		if e != nil {
-			return e
-		}
-
-		// check if key already in store, reject doubles
-		if postBkt.Get([]byte(comment.ID)) != nil {
-			return errors.Errorf("key %s already in store", comment.ID)
-		}
-
-		// serialize comment to json []byte for bolt and save
-		if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
-			return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
-		}
-
-		ref := b.makeRef(comment)
-
-		// add reference to comment to "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		e = lastBkt.Put(commentTs, ref)
-		if e != nil {
-			return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
-		}
-
-		// add reference to commentID to "users" bucket
-		userBkt, e := b.getUserBucket(tx, comment.User.ID)
-		if e != nil {
-			return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
-		}
-		// put into individual user's bucket with ts as a key
-		if e = userBkt.Put(commentTs, ref); e != nil {
-			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
-		}
-
-		// set info with the count for post url
-		if _, e = b.setInfo(tx, comment); e != nil {
-			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
-		}
-		return nil
-	})
-
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		return bucket.ForEach(func(k, v []byte) error {
-			comment := store.Comment{}
-			if e = json.Unmarshal(v, &comment); e != nil {
-				return errors.Wrap(e, "failed to unmarshal")
-			}
-			comments = append(comments, comment)
-			return nil
-		})
-	})
-
-	comments = SortComments(comments, sortFld)
-	return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		c := lastBkt.Cursor()
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			url, commentID, e := b.parseRef(v)
-			if e != nil {
-				return e
-			}
-			postBkt, e := b.getPostBucket(tx, url)
-			if e != nil {
-				return e
-			}
-
-			comment := store.Comment{}
-			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
-				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
-				continue
-			}
-			if comment.Deleted {
-				continue
-			}
-			comments = append(comments, comment)
-			if len(comments) >= max {
-				break
-			}
-		}
-		return nil
-	})
-
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return 0, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		var e error
-		count, e = b.count(tx, locator.URL, 0)
-		return e
-	})
-
-	return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		postsBkt := tx.Bucket([]byte(postsBucketName))
-
-		c := postsBkt.Cursor()
-		n := 0
-		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			n++
-			if skip > 0 && n <= skip {
-				continue
-			}
-			postURL := string(k)
-			infoBkt := tx.Bucket([]byte(infoBucketName))
-			info := store.PostInfo{}
-			if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
-				return errors.Wrapf(e, "can't load info for %s", postURL)
-			}
-			list = append(list, info)
-			if limit > 0 && len(list) >= limit {
-				break
-			}
-		}
-		return nil
-	})
-
-	return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return store.PostInfo{}, err
-	}
-
-	info := store.PostInfo{}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		infoBkt := tx.Bucket([]byte(infoBucketName))
-		if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
-			return errors.Wrapf(e, "can't load info for %s", locator.URL)
-		}
-		return nil
-	})
-
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if b.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-	commentRefs := []string{}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	if limit == 0 || limit > userLimit {
-		limit = userLimit
-	}
-
-	// get list of references to comments
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-
-		c := userIDBkt.Cursor()
-		skipComments := 0
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			if len(commentRefs) >= limit {
-				break
-			}
-			if skip > 0 && skipComments < skip {
-				skipComments++
-				continue
-			}
-			commentRefs = append(commentRefs, string(v))
-		}
-		return nil
-	})
-
-	if err != nil {
-		return comments, err
-	}
-
-	// retrieve comments for refs
-	for _, v := range commentRefs {
-		url, commentID, errParse := b.parseRef([]byte(v))
-		if errParse != nil {
-			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
-		}
-		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
-			comments = append(comments, c)
-		}
-	}
-
-	return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return 0, err
-	}
-	count := 0
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-		stats := userIDBkt.Stats()
-		count = stats.KeyN
-		return nil
-	})
-	return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return comment, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.load(bucket, []byte(commentID), &comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
-	if curComment, err := b.Get(locator, comment.ID); err == nil {
-		// preserve immutable fields
-		comment.ParentID = curComment.ParentID
-		comment.Locator = curComment.Locator
-		comment.Timestamp = curComment.Timestamp
-		comment.User = curComment.User
-	}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.save(bucket, []byte(comment.ID), comment)
-	})
-}
-
-// Close boltdb store
-func (b *BoltDB) Close() error {
-	errs := new(multierror.Error)
-	for site, db := range b.dbs {
-		err := errors.Wrapf(db.Close(), "can't close site %s", site)
-		errs = multierror.Append(errs, err)
-	}
-	return errs.ErrorOrNil()
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res := postsBkt.Bucket([]byte(postURL))
-	if res == nil {
-		return nil, errors.Errorf("no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
-	if err != nil {
-		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
-	usersBkt := tx.Bucket([]byte(userBucketName))
-	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
-	if e != nil {
-		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
-	}
-	return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
-	if value == nil {
-		return errors.Errorf("can't save nil value for %s", key)
-	}
-	jdata, jerr := json.Marshal(value)
-	if jerr != nil {
-		return errors.Wrap(jerr, "can't marshal comment")
-	}
-	if err = bkt.Put(key, jdata); err != nil {
-		return errors.Wrapf(err, "failed to save key %s", key)
-	}
-	return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
-	value := bkt.Get(key)
-	if value == nil {
-		return errors.Errorf("no value for %s", key)
-	}
-
-	if err := json.Unmarshal(value, &res); err != nil {
-		return errors.Wrap(err, "failed to unmarshal")
-	}
-	return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
-		info = store.PostInfo{}
-	}
-	if val == 0 { // get current count, don't update
-		return info.Count, nil
-	}
-	info.Count += val
-
-	return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
-		info = store.PostInfo{
-			Count:   0,
-			URL:     comment.Locator.URL,
-			FirstTS: comment.Timestamp,
-			LastTS:  comment.Timestamp,
-		}
-	}
-	info.Count++
-	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
-	if res, ok := b.dbs[siteID]; ok {
-		return res, nil
-	}
-	return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
-	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
-	elems := strings.Split(string(val), "!!")
-	if len(elems) != 2 {
-		return "", "", errors.Errorf("invalid reference value %s", string(val))
-	}
-	return elems[0], elems[1], nil
-}

backend/app/store/engine/bolt_accessor_test.go

@@ -1,384 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
-	assert.Equal(t, "key id-1 already in store", err.Error())
-
-	_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-
-	assert.NoError(t, b.Close())
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	comment := store.Comment{
-		ID:        "id-ro",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	err := b.SetReadOnly(comment.Locator, true)
-	require.Nil(t, err)
-
-	_, err = b.Create(comment)
-	assert.NotNil(t, err)
-	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
-	err = b.SetReadOnly(comment.Locator, false)
-	require.Nil(t, err)
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err)
-
-	_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = b.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = b.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = b.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	_, err = b.Last("bad", 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Count(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-
-	_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
-	b, teardown := prep(t) // two comments for https://radio-t.com
-	defer teardown()
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
-	res, err := b.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = b.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = b.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = b.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	_, err = b.List("bad", 1, 1)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
-	b, teardown := prep(t) // two comments for https://radio-t.com
-	defer teardown()
-
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	res, err := b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	_, err = b.User("bad", "user1", 1, 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.User("radio-t", "userZ", 1, 0)
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
-	os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-
-	defer func() {
-		require.NoError(t, b.Close())
-		os.Remove(testDb)
-	}()
-
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err = b.Create(c)
-		require.Nil(t, err)
-	}
-
-	// get all comments
-	res, err := b.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = b.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
-	var b, teardown = prep(t)
-	defer teardown()
-
-	count, err := b.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	_, err = b.UserCount("bad", "user1")
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.UserCount("radio-t", "userZ")
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
-	b := BoltDB{}
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	res := b.makeRef(comment)
-	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
-	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/2", url)
-	assert.Equal(t, "12345", id)
-
-	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
-	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
-	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) (b *BoltDB, teardown func()) {
-	os.Remove(testDb)
-
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b = boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	teardown = func() {
-		require.NoError(t, b.Close())
-		os.Remove(testDb)
-	}
-	return b, teardown
-}

backend/app/store/engine/bolt_admin.go

@@ -1,339 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"time"
-
-	bolt "github.com/coreos/bbolt"
-	log "github.com/go-pkgz/lgr"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		comment := store.Comment{}
-		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
-			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
-		}
-		// set deleted status and clear fields
-		comment.SetDeleted(mode)
-
-		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
-			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
-		}
-
-		// delete from "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err = lastBkt.Delete([]byte(commentID)); err != nil {
-			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
-		}
-
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
-		return nil
-	})
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
-	// delete top-level buckets
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		for _, bktName := range toDelete {
-
-			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
-			}
-			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-			}
-		}
-		return nil
-	})
-
-	return errors.Wrapf(err, "failed to delete top level buckets from site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// get list of all comments outside of transaction loop
-	posts, err := b.List(siteID, 0, 0)
-	if err != nil {
-		return err
-	}
-
-	type commentInfo struct {
-		locator   store.Locator
-		commentID string
-	}
-
-	// get list of commentID for all user's comment
-	comments := []commentInfo{}
-	for _, postInfo := range posts {
-		err = bdb.View(func(tx *bolt.Tx) error {
-			postsBkt := tx.Bucket([]byte(postsBucketName))
-			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
-				comment := store.Comment{}
-				if err = json.Unmarshal(commentVal, &comment); err != nil {
-					return errors.Wrap(err, "failed to unmarshal")
-				}
-				if comment.User.ID == userID {
-					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
-				}
-				return nil
-			})
-			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
-	// delete collected comments
-	for _, ci := range comments {
-		if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
-			return errors.Wrapf(err, "failed to delete comment %+v", ci)
-		}
-	}
-
-	//  delete  user bucket
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		if usersBkt != nil {
-			if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
-				return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "can't delete user meta")
-	}
-
-	if len(comments) == 0 {
-		return errors.Errorf("unknown user %s", userID)
-	}
-
-	return err
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		switch status {
-		case true:
-			val := time.Now().AddDate(100, 0, 0).Format(tsNano) // permanent is 100 year
-			if ttl > 0 {
-				val = time.Now().Add(ttl).Format(tsNano)
-			}
-			if e := bucket.Put([]byte(userID), []byte(val)); e != nil {
-				return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
-			}
-		}
-		return nil
-	})
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		val := bucket.Get([]byte(userID))
-		if val == nil {
-			blocked = false
-			return nil
-		}
-
-		until, e := time.Parse(tsNano, string(val))
-		if e != nil {
-			blocked = false
-			return nil
-		}
-		blocked = time.Now().Before(until)
-		return nil
-	})
-	return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
-			if errParse != nil {
-				return errors.Wrap(errParse, "can't parse block ts")
-			}
-			if time.Now().Before(ts) {
-				// get user name from comment user section
-				userName := ""
-				userComments, errUser := b.User(siteID, string(k), 1, 0)
-				if errUser == nil && len(userComments) > 0 {
-					userName = userComments[0].User.Name
-				}
-				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
-			}
-			return nil
-		})
-	})
-
-	return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
-			}
-		case false:
-			if e := bucket.Delete([]byte(locator.URL)); e != nil {
-				return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
-			}
-		}
-		return nil
-	})
-}
-
-// IsReadOnly checks if post in RO mode
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		ro = bucket.Get([]byte(locator.URL)) != nil
-		return nil
-	})
-	return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set verified status for %s", userID)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean verified status for %s", userID)
-			}
-		}
-		return nil
-	})
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		verified = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return verified
-}
-
-// Verified returns list of verified userIDs
-func (b *BoltDB) Verified(siteID string) (ids []string, err error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(verifiedBucketName))
-		_ = usersBkt.ForEach(func(k, _ []byte) error {
-			ids = append(ids, string(k))
-			return nil
-		})
-		return nil
-	})
-	return ids, err
-}

backend/app/store/engine/bolt_admin_test.go

@@ -1,246 +0,0 @@
-package engine
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	count, err := b.Count(loc)
-	require.NoError(t, err)
-	assert.Equal(t, 2, count, "count=2 initially")
-
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	count, err = b.Count(loc)
-	require.NoError(t, err)
-	assert.Equal(t, 1, count)
-
-	err = b.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	err = b.DeleteAll("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	err := b.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	_, err = b.User("radio-t", "user1", 5, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	err = b.DeleteUser("radio-t-bad", "user1")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", false, 0))
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.EqualError(t, b.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
-	assert.NoError(t, b.SetBlock("radio-t", "userX", false, 0))
-
-	assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockUserWithTTL(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 50*time.Millisecond))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-	time.Sleep(50 * time.Millisecond)
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true, 0))
-	assert.NoError(t, b.SetBlock("radio-t", "user2", true, 50*time.Millisecond))
-	assert.NoError(t, b.SetBlock("radio-t", "user3", false, 0))
-
-	ids, err := b.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	time.Sleep(50 * time.Millisecond)
-	ids, err = b.Blocked("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-
-	_, err = b.Blocked("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-
-	b, teardown := prep(t)
-	defer teardown()
-
-	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, b.SetVerified("radio-t", "u1", false))
-	assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u2", true))
-	assert.NoError(t, b.SetVerified("radio-t", "u3", false))
-
-	ids, err := b.Verified("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
-	_, err = b.Verified("radio-t-bad")
-	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
-}

backend/app/store/engine/bolt_test.go

@@ -0,0 +1,845 @@
+package engine
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestBoltDB_CreateAndFind(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	var bb Interface = b
+	_ = bb
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+	t.Log(res[0].ID)
+
+	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	assert.NotNil(t, err)
+	assert.Equal(t, "key id-1 already in store", err.Error())
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, Sort: "time"}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+	assert.NoError(t, b.Close())
+}
+
+func TestBoltDB_CreateFailedReadOnly(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.NotNil(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = FlagRequest{Locator: comment.Locator, Flag: ReadOnly, Update: FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestBoltDB_Get(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.NotNil(t, err)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_Update(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	comment.Locator.SiteID = "radio-t"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
+}
+
+func TestBoltDB_FindLast(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_FindLastSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindInPostSince(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_FindForUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	_, err = b.Find(req)
+	assert.EqualError(t, err, `no comments for user userZ in store`)
+}
+
+func TestBoltDB_FindForUserPagination(t *testing.T) {
+	_ = os.Remove(testDb)
+	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.Nil(t, err)
+
+	defer func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("id-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err = b.Create(c)
+		require.Nil(t, err)
+	}
+
+	// get all comments
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 5, len(res))
+	assert.Equal(t, "id-199", res[0].ID)
+	assert.Equal(t, "id-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, "id-189", res[0].ID)
+	assert.Equal(t, "id-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 5, len(res))
+	assert.Equal(t, "id-4", res[0].ID)
+	assert.Equal(t, "id-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestBoltDB_CountPost(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltDB_CountUser(t *testing.T) {
+	var b, teardown = prep(t)
+	defer teardown()
+
+	req := FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	_, err = b.Count(req)
+	assert.EqualError(t, err, `no comments for user userZ in store for radio-t site`)
+}
+
+func TestBoltDB_InfoPost(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.NotNil(t, err)
+
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.NotNil(t, err)
+
+	fr := FlagRequest{Flag: ReadOnly, Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestBoltDB_InfoList(t *testing.T) {
+	b, teardown := prep(t) // two comments for https://radio-t.com
+	defer teardown()
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.Nil(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
+
+	req := InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	_, err = b.Info(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBolt_FlagBlockedUser(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}, UserID: "user1", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "userX", Update: FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err, "non-existing user can't be blocked")
+
+	req = FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked on wrong site")
+}
+
+func TestBolt_FlagReadOnlyPost(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	req := FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.Equal(t, true, val)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "bad", URL: "url-1"}, Flag: ReadOnly, Update: FlagFalse}
+	_, err = b.Flag(req)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	req = FlagRequest{Locator: store.Locator{SiteID: "radio-t-bad", URL: "url-1"}, Flag: ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro on wrong site")
+}
+
+func TestBolt_FlagVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	isVerified := func(site, user string) bool {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.EqualError(t, setVerified("bad", "u1", FlagTrue), `site "bad" not found`)
+	assert.NoError(t, setVerified("radio-t", "u1xyz", FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", FlagFalse))
+}
+
+func TestBolt_FlagListVerified(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+
+	setVerified := func(site, user string, status FlagStatus) error {
+		req := FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", FlagTrue))
+	ids, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	_, err = b.ListFlags(FlagRequest{Flag: Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.Error(t, err, "site \"radio-t-bad\" not found", "fail on wrong site")
+}
+
+func TestBolt_FlagListBlocked(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	setBlocked := func(site, user string, status FlagStatus, ttl time.Duration) error {
+		req := FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status, TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", FlagFalse, 0))
+
+	vv, err := b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	assert.Equal(t, 2, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+	assert.Equal(t, "user2", blockedList[1].ID)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	assert.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	_, err = b.ListFlags(FlagRequest{Flag: Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.EqualError(t, err, `site "bad" not found`)
+
+}
+
+func TestBolt_DeleteComment(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 2, count, "count=2 initially")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.NotNil(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
+}
+
+func TestBolt_DeleteHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	reqReq := FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestBolt_DeleteAll(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	delReq := DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	delReq = DeleteRequest{Locator: store.Locator{SiteID: "bad"}}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `site "bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserHard(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.EqualError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltAdmin_DeleteUserSoft(t *testing.T) {
+
+	b, teardown := prep(t)
+	defer teardown()
+
+	err := b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	assert.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+
+	err = b.Delete(DeleteRequest{Locator: store.Locator{SiteID: "radio-t-bad"}, UserID: "user1"})
+	assert.EqualError(t, err, `site "radio-t-bad" not found`)
+}
+
+func TestBoltDB_ref(t *testing.T) {
+	b := BoltDB{}
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	res := b.makeRef(comment)
+	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
+
+	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
+	assert.NoError(t, err)
+	assert.Equal(t, "https://radio-t.com/2", url)
+	assert.Equal(t, "12345", id)
+
+	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
+	assert.NotNil(t, err)
+}
+
+func TestBoltDB_NewFailed(t *testing.T) {
+	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
+	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) (b *BoltDB, teardown func()) {
+	_ = os.Remove(testDb)
+
+	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
+	assert.Nil(t, err)
+	b = boltStore
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	teardown = func() {
+		require.NoError(t, b.Close())
+		_ = os.Remove(testDb)
+	}
+	return b, teardown
+}
+
+func getReq(locator store.Locator, commentID string) GetRequest {
+	return GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/app/store/engine/engine.go

@@ -1,6 +1,7 @@
+package engine
+
 // Package engine defines interfaces each supported storage should implement.
 // Includes default implementation with boltdb
-package engine
 
 import (
 	"sort"
@@ -10,50 +11,82 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 )
 
-//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go"
+// NOTE: mockery works from linked to go-path and with GOFLAGS='-mod=vendor' go generate
+//go:generate sh -c "mockery -inpkg -name Interface -print > /tmp/engine-mock.tmp && mv /tmp/engine-mock.tmp engine_mock.go"
 
-// Interface combines all store interfaces
+// Interface defines methods provided by low-level storage engine
 type Interface interface {
-	Accessor
+	Create(comment store.Comment) (commentID string, err error) // create new comment, avoid dups by id
-	Admin
+	Update(comment store.Comment) error                         // update comment, mutable parts only
+	Get(req GetRequest) (store.Comment, error)                  // get comment by id
+	Find(req FindRequest) ([]store.Comment, error)              // find comments for locator or site
+	Info(req InfoRequest) ([]store.PostInfo, error)             // get post(s) meta info
+	Count(req FindRequest) (int, error)                         // get count for post or user
+	Delete(req DeleteRequest) error                             // delete post(s) by id or by userID
+	Flag(req FlagRequest) (bool, error)                         // set and get flags
+	ListFlags(req FlagRequest) ([]interface{}, error)           // get list of flagged keys, like blocked & verified user
+	Close() error                                               // close storage engine
 }
 
-// UserRequest is the request send to get comments by user
+// GetRequest is the input for Get func
-type UserRequest struct {
+type GetRequest struct {
-	SiteID string
+	Locator   store.Locator `json:"locator"`
-	UserID string
+	CommentID string        `json:"comment_id"`
-	Limit  int
-	Skip   int
 }
 
-// Accessor defines all usual access ops avail for regular user
+// FindRequest is the input for all find operations
-type Accessor interface {
+type FindRequest struct {
-	Create(comment store.Comment) (commentID string, err error)           // create new comment, avoid dups by id
+	Locator store.Locator `json:"locator"`           // lack of URL means site operation
-	Get(locator store.Locator, commentID string) (store.Comment, error)   // get comment by id
+	UserID  string        `json:"user_id,omitempty"` // presence of UserID treated as user-related find
-	Put(locator store.Locator, comment store.Comment) error               // update comment, mutable parts only
+	Sort    string        `json:"sort,omitempty"`    // sort order with +/-field syntax
-	Find(locator store.Locator, sort string) ([]store.Comment, error)     // find comments for locator
+	Since   time.Time     `json:"since,omitempty"`   // time limit for found results
-	Last(siteID string, limit int) ([]store.Comment, error)               // last comments for given site, sorted by time
+	Limit   int           `json:"limit,omitempty"`
-	User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
+	Skip    int           `json:"skip,omitempty"`
-	UserCount(siteID, userID string) (int, error)                         // comments count by user
-	Count(locator store.Locator) (int, error)                             // number of comments for the post
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)    // list of commented posts
-	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)  // get post info
-	Close() error                                                         // close/stop engine
 }
 
-// Admin defines all store ops avail for admin only
+// InfoRequest is the input of Info operation used to get meta data about posts
-type Admin interface {
+type InfoRequest struct {
-	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
+	Locator     store.Locator `json:"locator"`
-	DeleteAll(siteID string) error                                               // delete all data from site
+	Limit       int           `json:"limit,omitempty"`
-	DeleteUser(siteID string, userID string) error                               // remove all comments from user
+	Skip        int           `json:"skip,omitempty"`
-	SetBlock(siteID string, userID string, status bool, ttl time.Duration) error // block or unblock user with TTL (0-permanent)
+	ReadOnlyAge int           `json:"ro_age,omitempty"`
-	IsBlocked(siteID string, userID string) bool                                 // check if user blocked
+}
-	Blocked(siteID string) ([]store.BlockedUser, error)                          // get list of blocked users
+
-	SetReadOnly(locator store.Locator, status bool) error                        // set/reset read-only flag
+// DeleteRequest is the input for all delete operations (comments, sites, users)
-	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
+type DeleteRequest struct {
-	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
+	Locator    store.Locator    `json:"locator"` // lack of URL means site operation
-	IsVerified(siteID string, userID string) bool                                // check verified status
+	CommentID  string           `json:"comment_id,omitempty"`
-	Verified(siteID string) ([]string, error)                                    // list of verified user ids
+	UserID     string           `json:"user_id,omitempty"`
+	DeleteMode store.DeleteMode `json:"del_mode"`
+}
+
+// Flag defines type of binary attribute
+type Flag string
+
+// FlagStatus represents values of the flag update
+type FlagStatus int
+
+// enum of update values
+const (
+	FlagNonSet FlagStatus = 0
+	FlagTrue   FlagStatus = 1
+	FlagFalse  FlagStatus = -1
+)
+
+// Enum of all flags
+const (
+	ReadOnly = Flag("readonly")
+	Verified = Flag("verified")
+	Blocked  = Flag("blocked")
+)
+
+// FlagRequest is the input for both get/set for flags, like blocked, verified and so on
+type FlagRequest struct {
+	Flag    Flag          `json:"flag"`              // flag type
+	Locator store.Locator `json:"locator"`           // post locator
+	UserID  string        `json:"user_id,omitempty"` // for flags setting user status
+	Update  FlagStatus    `json:"update,omitempty"`  // if FlagNonSet it will be get op, if set will set the value
+	TTL     time.Duration `json:"ttl,omitempty"`     // ttl for time-sensitive flags only, like blocked for some period
 }
 
 const (

backend/app/store/engine/engine_mock.go

@@ -0,0 +1,205 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+package engine
+
+import mock "github.com/stretchr/testify/mock"
+import store "github.com/umputun/remark/backend/app/store"
+
+// MockInterface is an autogenerated mock type for the Interface type
+type MockInterface struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *MockInterface) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Count provides a mock function with given fields: req
+func (_m *MockInterface) Count(req FindRequest) (int, error) {
+	ret := _m.Called(req)
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func(FindRequest) int); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Create provides a mock function with given fields: comment
+func (_m *MockInterface) Create(comment store.Comment) (string, error) {
+	ret := _m.Called(comment)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(store.Comment) string); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(store.Comment) error); ok {
+		r1 = rf(comment)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Delete provides a mock function with given fields: req
+func (_m *MockInterface) Delete(req DeleteRequest) error {
+	ret := _m.Called(req)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(DeleteRequest) error); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Find provides a mock function with given fields: req
+func (_m *MockInterface) Find(req FindRequest) ([]store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.Comment
+	if rf, ok := ret.Get(0).(func(FindRequest) []store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.Comment)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FindRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Flag provides a mock function with given fields: req
+func (_m *MockInterface) Flag(req FlagRequest) (bool, error) {
+	ret := _m.Called(req)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(FlagRequest) bool); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Get provides a mock function with given fields: req
+func (_m *MockInterface) Get(req GetRequest) (store.Comment, error) {
+	ret := _m.Called(req)
+
+	var r0 store.Comment
+	if rf, ok := ret.Get(0).(func(GetRequest) store.Comment); ok {
+		r0 = rf(req)
+	} else {
+		r0 = ret.Get(0).(store.Comment)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(GetRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Info provides a mock function with given fields: req
+func (_m *MockInterface) Info(req InfoRequest) ([]store.PostInfo, error) {
+	ret := _m.Called(req)
+
+	var r0 []store.PostInfo
+	if rf, ok := ret.Get(0).(func(InfoRequest) []store.PostInfo); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]store.PostInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(InfoRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListFlags provides a mock function with given fields: req
+func (_m *MockInterface) ListFlags(req FlagRequest) ([]interface{}, error) {
+	ret := _m.Called(req)
+
+	var r0 []interface{}
+	if rf, ok := ret.Get(0).(func(FlagRequest) []interface{}); ok {
+		r0 = rf(req)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]interface{})
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(FlagRequest) error); ok {
+		r1 = rf(req)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Update provides a mock function with given fields: comment
+func (_m *MockInterface) Update(comment store.Comment) error {
+	ret := _m.Called(comment)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(store.Comment) error); ok {
+		r0 = rf(comment)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

backend/app/store/engine/mongo.go

@@ -1,377 +0,0 @@
-package engine
-
-import (
-	"time"
-
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	"github.com/go-pkgz/mongo"
-	multierror "github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-// Mongo implements engine interface
-type Mongo struct {
-	conn       *mongo.Connection
-	postWriter mongo.BufferedWriter
-}
-
-const (
-	mongoPosts     = "posts"
-	mongoMetaPosts = "meta_posts"
-	mongoMetaUsers = "meta_users"
-)
-
-type metaPost struct {
-	ID       string `bson:"_id"` // url
-	SiteID   string `bson:"site"`
-	ReadOnly bool   `bson:"read_only"`
-}
-
-type metaUser struct {
-	ID           string    `bson:"_id"` // user_id
-	SiteID       string    `bson:"site"`
-	Verified     bool      `bson:"verified"`
-	Blocked      bool      `bson:"blocked"`
-	BlockedUntil time.Time `bson:"blocked_until"`
-}
-
-// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
-// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
-func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
-	writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
-	result := Mongo{conn: conn, postWriter: writer}
-	err := result.prepare()
-	return &result, errors.Wrap(err, "failed to prepare mongo")
-}
-
-// Create new comment, write can be buffered and delayed.
-func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
-	// err = m.postWriter.Write(comment)
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Insert(&comment)
-	})
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).Sort(sortFld).All(&comments)
-	})
-	return comments, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
-		return coll.Find(query).One(&comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
-			bson.M{"$set": bson.M{
-				"text":    comment.Text,
-				"orig":    comment.Orig,
-				"score":   comment.Score,
-				"votes":   comment.Votes,
-				"pin":     comment.Pin,
-				"deleted": comment.Deleted,
-			}})
-	})
-}
-
-// Last returns up to max last comments for given siteID
-func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "delete": false}
-		return coll.Find(query).Sort("-time").Limit(max).All(&comments)
-	})
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (m *Mongo) Count(locator store.Locator) (count int, err error) {
-
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
-		count, err = coll.Find(query).Count()
-		return err
-	})
-	return count, e
-}
-
-// List returns list of all commented posts with counters
-func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-	list = []store.PostInfo{}
-
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": siteID}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-			{"$skip": skip},
-			{"$limit": limit},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	return list, errors.Wrap(err, "can't get list")
-}
-
-// Info returns time range and count for locator
-func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
-	list := []store.PostInfo{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		pipeline := coll.Pipe([]bson.M{
-			{"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
-			{"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
-			{"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
-				"first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
-		})
-		return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
-	})
-	if err != nil {
-		return info, err
-	}
-	if len(list) == 0 {
-		return info, errors.Errorf("can't load info for %s", locator.URL)
-	}
-	info = list[0]
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if m.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, nil
-}
-
-// User extracts all comments for given site and given userID
-func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		query := bson.M{"locator.site": siteID, "user.id": userID}
-		return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
-	})
-	return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
-}
-
-// UserCount returns number of comments for user
-func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
-	err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		var e error
-		count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
-		return e
-	})
-	return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
-	return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
-		return e
-	})
-}
-
-// IsReadOnly checks if post in RO
-func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
-	meta := metaPost{}
-	err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
-	})
-	return err == nil && meta.ReadOnly
-}
-
-// SetVerified makes user verified or reset the flag
-func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
-		return e
-	})
-}
-
-// IsVerified checks if user verified
-func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Verified
-}
-
-// Verified returns list of verified user IDs
-func (m *Mongo) Verified(siteID string) (ids []string, err error) {
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID, "verified": true}).All(&metas)
-	})
-	if err != nil {
-		return nil, err
-	}
-	for _, meta := range metas {
-		ids = append(ids, meta.ID)
-	}
-	return ids, nil
-}
-
-// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
-// block uses blocksBucketName with key=userID and val=TTL+now
-func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
-
-	until := time.Time{}
-	if status {
-		until = time.Now().AddDate(100, 0, 0) // permanent is 50year
-		if ttl > 0 {
-			until = time.Now().Add(ttl)
-		}
-	}
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		_, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
-			bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
-		return errors.Wrapf(e, "failed to set block for %s", userID)
-	})
-}
-
-// IsBlocked checks if user blocked
-func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
-	meta := metaUser{}
-	err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
-	})
-	return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
-}
-
-// Blocked get lists of blocked users for given site
-func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	metas := []metaUser{}
-	err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		return coll.Find(bson.M{"site": siteID,
-			"blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
-	})
-	if err != nil {
-		return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
-	}
-
-	for _, mu := range metas {
-		blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
-		if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
-			blockedUser.Name = ucc[0].User.Name
-		}
-		users = append(users, blockedUser)
-	}
-	return users, nil
-}
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-	comment := store.Comment{}
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
-		if e != nil {
-			return e
-		}
-		comment.SetDeleted(mode)
-		return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
-	})
-	return errors.Wrapf(err, "can't delete %s", commentID)
-}
-
-// DeleteAll removes all info about siteID
-func (m *Mongo) DeleteAll(siteID string) error {
-	err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		_, e := coll.RemoveAll(bson.M{"locator.site": siteID})
-		return e
-	})
-	return errors.Wrapf(err, "can't delete site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted".
-func (m *Mongo) DeleteUser(siteID string, userID string) error {
-	comments := []store.Comment{}
-	return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
-		if e != nil {
-			return e
-		}
-		for _, c := range comments {
-			if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
-				return e
-			}
-		}
-		return nil
-	})
-}
-
-// Close boltdb store
-func (m *Mongo) Close() error {
-	if m.postWriter != nil {
-		return m.postWriter.Close()
-	}
-	return nil
-}
-
-// prepare collections with all indexes
-func (m *Mongo) prepare() error {
-	errs := new(multierror.Error)
-	e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
-	})
-	if e != nil {
-		return e
-	}
-
-	return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
-		errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
-		errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
-		return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
-	})
-}
-
-func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
-	if limit <= 0 {
-		limit = 1000
-	}
-	if skip < 0 {
-		skip = 0
-	}
-	return q.Skip(skip).Limit(limit)
-}

backend/app/store/engine/mongo_test.go

@@ -1,594 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/go-pkgz/mongo"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/store"
-)
-
-func TestMongo_CreateAndFind(t *testing.T) {
-	var m Interface
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	require.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err, "reject dup")
-
-	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
-	assert.Nil(t, err)
-	assert.Equal(t, "id-3", id)
-	res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
-	assert.Nil(t, err)
-	require.Equal(t, 1, len(res))
-
-	assert.NoError(t, m.Close())
-}
-
-func TestMongo_Get(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err, "not found")
-}
-
-func TestMongo_Put(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = m.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = m.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `not found`)
-
-	err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `not found`)
-}
-
-func TestMongo_Last(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = m.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-}
-
-func TestMongo_Count(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-}
-
-func TestMongo_List(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := m.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
-
-	res, err := m.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = m.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = m.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = m.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = m.List("bad", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{}, res)
-}
-
-func TestMongo_Info(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := m.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-}
-
-func TestMongo_ReadOnly(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
-	assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestMongo_Verified(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, m.SetVerified("radio-t", "u1", true))
-	assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, m.SetVerified("radio-t", "u1", false))
-	assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-
-	assert.NoError(t, m.SetVerified("radio-t", "u1", true))
-	assert.NoError(t, m.SetVerified("radio-t", "u2", true))
-	assert.NoError(t, m.SetVerified("radio-t", "u3", false))
-
-	ids, err := m.Verified("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"u1", "u2"}, ids, "verified 2 ids")
-
-	ids, err = m.Verified("radio-t-bad")
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_GetForUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	res, err := m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = m.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = m.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	res, err = m.User("bad", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_GetForUserPagination(t *testing.T) {
-	m, skip := prepMongo(t, false)
-	if skip {
-		return
-	}
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err := m.Create(c)
-		require.Nil(t, err, c.ID)
-	}
-
-	// get all comments
-	res, err := m.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = m.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = m.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = m.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestMongo_BlockUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
-	assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
-	assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
-
-	assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestMongo_BlockUserWithTTL(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
-	assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
-	time.Sleep(500 * time.Millisecond)
-	assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
-}
-
-func TestMongo_GetForUserCounter(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	count, err := m.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	count, err = m.UserCount("bad", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 0, count)
-}
-
-func TestMongo_BlockList(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
-	assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
-	assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
-
-	ids, err := m.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	time.Sleep(500 * time.Millisecond)
-	ids, err = m.Blocked("radio-t")
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-
-	ids, err = m.Blocked("bad")
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(ids))
-}
-
-func TestMongo_Delete(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	err = m.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `can't delete id-1: not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = m.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `can't delete id-1: not found`)
-}
-
-func TestMongo_DeleteHard(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestMongo_DeleteAll(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = m.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-}
-
-func TestMongo_DeleteUser(t *testing.T) {
-	m, skip := prepMongo(t, true) // adds two comments
-	if skip {
-		return
-	}
-	err := m.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := m.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	cc, err := m.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err, "no comments for user user1 in store")
-	assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
-
-	comments, err := m.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-}
-
-func TestMongo_Parallel(t *testing.T) {
-	var m Interface
-	var skip bool
-	m, skip = prepMongoBuffered(t) // buffered engine, no comments
-	if skip {
-		return
-	}
-	go func() {
-		for i := 0; i < 100; i++ {
-			_, err := m.Create(store.Comment{
-				ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-			require.Nil(t, err)
-			time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
-		}
-	}()
-
-	for {
-		time.Sleep(10 * time.Millisecond)
-		res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-		assert.Nil(t, err)
-		if len(res) == 100 {
-			break
-		}
-	}
-}
-
-func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
-	conn, err := mongo.MakeTestConnection(t)
-	if err != nil {
-		return nil, true
-	}
-	mongo.RemoveTestCollection(t, conn)
-
-	m, err := NewMongo(conn, 1, 0*time.Microsecond)
-	require.Nil(t, err)
-
-	mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	if writeRecs {
-		_, err = m.Create(comment)
-		assert.Nil(t, err)
-	}
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	if writeRecs {
-		_, err = m.Create(comment)
-		assert.Nil(t, err)
-	}
-
-	return m, false
-}
-
-func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
-	conn, err := mongo.MakeTestConnection(t)
-	if err != nil {
-		return nil, true
-	}
-	mongo.RemoveTestCollection(t, conn)
-
-	m, err := NewMongo(conn, 10, 10*time.Millisecond)
-	mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
-
-	require.Nil(t, err)
-	return m, false
-}

backend/app/store/engine/remote.go

@@ -0,0 +1,104 @@
+package engine
+
+import (
+	"encoding/json"
+
+	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// RPC implements remote engine and delegates all Calls to remote http server
+type RPC struct {
+	rpc.Client
+}
+
+// Create comment and return ID
+func (r *RPC) Create(comment store.Comment) (commentID string, err error) {
+
+	resp, err := r.Call("store.create", comment)
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(*resp.Result, &commentID)
+	return commentID, err
+}
+
+// Get comment by ID
+func (r *RPC) Get(req GetRequest) (comment store.Comment, err error) {
+	resp, err := r.Call("store.get", req)
+	if err != nil {
+		return store.Comment{}, err
+	}
+
+	err = json.Unmarshal(*resp.Result, &comment)
+	return comment, err
+}
+
+// Update comment, mutable parts only
+func (r *RPC) Update(comment store.Comment) error {
+	_, err := r.Call("store.update", comment)
+	return err
+}
+
+// Find comments for locator
+func (r *RPC) Find(req FindRequest) (comments []store.Comment, err error) {
+	resp, err := r.Call("store.find", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &comments)
+	return comments, err
+}
+
+// Info returns post(s) meta info
+func (r *RPC) Info(req InfoRequest) (info []store.PostInfo, err error) {
+	resp, err := r.Call("store.info", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &info)
+	return info, err
+}
+
+// Flag sets and gets flags
+func (r *RPC) Flag(req FlagRequest) (status bool, err error) {
+	resp, err := r.Call("store.flag", req)
+	if err != nil {
+		return false, err
+	}
+	err = json.Unmarshal(*resp.Result, &status)
+	return status, err
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+func (r *RPC) ListFlags(req FlagRequest) (list []interface{}, err error) {
+	resp, err := r.Call("store.list_flags", req)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(*resp.Result, &list)
+	return list, err
+}
+
+// Count gets comments count by user or site
+func (r *RPC) Count(req FindRequest) (count int, err error) {
+	resp, err := r.Call("store.count", req)
+	if err != nil {
+		return 0, err
+	}
+	err = json.Unmarshal(*resp.Result, &count)
+	return count, err
+}
+
+// Delete post(s) by id or by userID
+func (r *RPC) Delete(req DeleteRequest) error {
+	_, err := r.Call("store.delete", req)
+	return err
+}
+
+// Close storage engine
+func (r *RPC) Close() error {
+	_, err := r.Call("store.close")
+	return err
+}

backend/app/store/engine/remote_test.go

@@ -0,0 +1,178 @@
+package engine
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/rpc"
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestRemote_Create(t *testing.T) {
+	ts := testServer(t, `{"method":"store.create","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`,
+		`{"result":"12345","id":1}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	var eng Interface = &c
+	_ = eng
+
+	res, err := c.Create(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site"},
+		Text: "msg"})
+	assert.NoError(t, err)
+	assert.Equal(t, "12345", res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_Get(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"result":{"id":"123","pid":"","text":"msg","delete":true}}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	res, err := c.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, store.Comment{ID: "123", Text: "msg", Deleted: true}, res)
+	t.Logf("%v %T", res, res)
+}
+
+func TestRemote_GetWithErrorResult(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, `{"error":"failed"}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "failed")
+}
+
+func TestRemote_GetWithErrorDecode(t *testing.T) {
+	ts := testServer(t, `{"method":"store.get","params":{"locator":{"url":"http://example.com/url"},"comment_id":"site"},"id":1}`, ``)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "failed to decode response for store.get: EOF")
+}
+
+func TestRemote_GetWithErrorRemote(t *testing.T) {
+	c := RPC{Client: rpc.Client{API: "http://127.0.0.2", Client: http.Client{Timeout: 10 * time.Millisecond}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.NotNil(t, err)
+	assert.True(t, strings.Contains(err.Error(), "remote call failed for store.get:"), err.Error())
+}
+
+func TestRemote_FailedStatus(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		t.Logf("req: %s", string(body))
+		w.WriteHeader(400)
+	}))
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	req := GetRequest{Locator: store.Locator{URL: "http://example.com/url"}, CommentID: "site"}
+	_, err := c.Get(req)
+	assert.EqualError(t, err, "bad status 400 Bad Request for store.get")
+}
+
+func TestRemote_Update(t *testing.T) {
+	ts := testServer(t, `{"method":"store.update","params":{"id":"123","pid":"","text":"msg","user":{"name":"","id":"","picture":"","admin":false},"locator":{"site":"site123","url":"http://example.com/url"},"score":0,"vote":0,"time":"0001-01-01T00:00:00Z"},"id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	err := c.Update(store.Comment{ID: "123", Locator: store.Locator{URL: "http://example.com/url", SiteID: "site123"},
+		Text: "msg"})
+	assert.NoError(t, err)
+
+}
+
+func TestRemote_Find(t *testing.T) {
+	ts := testServer(t, `{"method":"store.find","params":{"locator":{"url":"http://example.com/url"},"sort":"-time","since":"0001-01-01T00:00:00Z","limit":10},"id":1}`, `{"result":[{"text":"1"},{"text":"2"}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Find(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}, Sort: "-time", Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, []store.Comment{{Text: "1"}, {Text: "2"}}, res)
+}
+
+func TestRemote_Info(t *testing.T) {
+	ts := testServer(t, `{"method":"store.info","params":{"locator":{"url":"http://example.com/url"},"limit":10,"skip":5,"ro_age":10},"id":1}`, `{"result":[{"url":"u1","count":22},{"url":"u2","count":33}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Info(InfoRequest{Locator: store.Locator{URL: "http://example.com/url"},
+		Limit: 10, Skip: 5, ReadOnlyAge: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "u1", Count: 22}, {URL: "u2", Count: 33}}, res)
+}
+
+func TestRemote_Flag(t *testing.T) {
+	ts := testServer(t, `{"method":"store.flag","params":{"flag":"verified","locator":{"url":"http://example.com/url"}},"id":1}`, `{"result":false}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Flag(FlagRequest{Locator: store.Locator{URL: "http://example.com/url"}, Flag: Verified})
+	assert.NoError(t, err)
+	assert.Equal(t, false, res)
+}
+
+func TestRemote_ListFlag(t *testing.T) {
+	ts := testServer(t, `{"method":"store.list_flags","params":{"flag":"blocked","locator":{"site":"site_id","url":""}},"id":1}`, `{"result":[{"ID":"id1"},{"ID":"id2"}]}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	res, err := c.ListFlags(FlagRequest{Locator: store.Locator{SiteID: "site_id"}, Flag: Blocked})
+	assert.NoError(t, err)
+	assert.Equal(t, []interface{}{map[string]interface{}{"ID": "id1"}, map[string]interface{}{"ID": "id2"}}, res)
+}
+
+func TestRemote_Count(t *testing.T) {
+	ts := testServer(t, `{"method":"store.count","params":{"locator":{"url":"http://example.com/url"},"since":"0001-01-01T00:00:00Z"},"id":1}`, `{"result":11}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	res, err := c.Count(FindRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 11, res)
+}
+
+func TestRemote_Delete(t *testing.T) {
+	ts := testServer(t, `{"method":"store.delete","params":{"locator":{"url":"http://example.com/url"},"del_mode":0},"id":1}`,
+		`{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+
+	err := c.Delete(DeleteRequest{Locator: store.Locator{URL: "http://example.com/url"}})
+	assert.NoError(t, err)
+}
+
+func TestRemote_Close(t *testing.T) {
+	ts := testServer(t, `{"method":"store.close","id":1}`, `{}`)
+	defer ts.Close()
+	c := RPC{Client: rpc.Client{API: ts.URL, Client: http.Client{}}}
+	err := c.Close()
+	assert.NoError(t, err)
+}
+
+func testServer(t *testing.T, req, resp string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		assert.Equal(t, req, string(body))
+		t.Logf("req: %s", string(body))
+		fmt.Fprintf(w, resp)
+	}))
+}

backend/app/store/formatter.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
+	bf "gopkg.in/russross/blackfriday.v2"
 )
 
 // CommentFormatter implements all generic formatting ops on comment
@@ -40,10 +40,16 @@ func (f *CommentFormatter) Format(c Comment) Comment {
 
 // FormatText converts text with markdown processor, applies external converters and shortens links
 func (f *CommentFormatter) FormatText(txt string) (res string) {
-	mdExt := blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
+	mdExt := bf.NoIntraEmphasis | bf.Tables | bf.FencedCode |
-		blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
+		bf.Strikethrough | bf.SpaceHeadings | bf.HardLineBreak |
-		blackfriday.BackslashLineBreak | blackfriday.Autolink
+		bf.BackslashLineBreak | bf.Autolink
-	res = string(blackfriday.Run([]byte(txt), blackfriday.WithExtensions(mdExt)))
+
+	rend := bf.NewHTMLRenderer(bf.HTMLRendererParameters{
+		Flags: bf.Smartypants | bf.SmartypantsFractions | bf.SmartypantsDashes | bf.SmartypantsAngledQuotes,
+	})
+
+	res = string(bf.Run([]byte(txt), bf.WithExtensions(mdExt), bf.WithRenderer(rend)))
+	res = f.unEscape(res)
 
 	for _, conv := range f.converters {
 		res = conv.Convert(res)
@@ -85,3 +91,16 @@ func (f *CommentFormatter) shortenAutoLinks(commentHTML string, max int) (resHTM
 	}
 	return resHTML
 }
+
+func (f *CommentFormatter) unEscape(txt string) (res string) {
+	elems := []struct {
+		from, to string
+	}{
+		{`—`, "—"},
+	}
+	res = txt
+	for _, e := range elems {
+		res = strings.Replace(res, e.from, e.to, -1)
+	}
+	return res
+}

backend/app/store/formatter_test.go

@@ -14,17 +14,23 @@ func (m mockConverter) Convert(text string) string { return text + "!converted"
 func TestFormatter_FormatText(t *testing.T) {
 	tbl := []struct {
 		in, out string
+		name    string
 	}{
-		{"", "!converted"},
+		{"", "!converted", "empty"},
-		{"12345 abc", "<p>12345 abc</p>\n!converted"},
+		{"12345 abc", "<p>12345 abc</p>\n!converted", "simple"},
-		{"**xyz** _aaa_", "<p><strong>xyz</strong> <em>aaa</em></p>\n!converted"},
+		{"**xyz** _aaa_ - \"sfs\"", "<p><strong>xyz</strong> <em>aaa</em> – «sfs»</p>\n!converted", "format"},
 		{
-			"http://127.0.0.1/some-long-link/12345/678901234567890", "<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0.1/some-long-link/12345/6789012...</a></p>\n!converted",
+			"http://127.0.0.1/some-long-link/12345/678901234567890",
+			"<p><a href=\"http://127.0.0.1/some-long-link/12345/678901234567890\">http://127.0.0." +
+				"1/some-long-link/12345/6789012...</a></p>\n!converted", "links",
 		},
+		{"&mdash; not translated #354", "<p>— not translated #354</p>\n!converted", "mdash"},
 	}
 	f := NewCommentFormatter(mockConverter{})
-	for n, tt := range tbl {
+	for _, tt := range tbl {
-		assert.Equal(t, tt.out, f.FormatText(tt.in), "check #%d", n)
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.out, f.FormatText(tt.in))
+		})
 	}
 }
 

backend/app/store/image/fs_store.go

@@ -122,7 +122,7 @@ func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
 		return nil
 	}
 
-	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(f.Staging, func(fpath string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -131,8 +131,10 @@ func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
 		}
 		age := time.Since(info.ModTime())
 		if age > ttl {
-			log.Printf("[INFO] remove staging image %s, age %v", path, age)
+			log.Printf("[INFO] remove staging image %s, age %v", fpath, age)
-			return os.Remove(path)
+			rmErr := os.Remove(fpath)
+			_ = os.Remove(path.Dir(fpath)) // try to remove directory
+			return rmErr
 		}
 		return nil
 	})

backend/app/store/image/fs_store_test.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 	"testing"
@@ -268,6 +269,9 @@ func TestFsStore_Cleanup(t *testing.T) {
 
 	_, err = os.Stat(img1)
 	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(path.Dir(img1))
+	assert.NotNil(t, err, "no dir %s on staging anymore", path.Dir(img1))
+
 	_, err = os.Stat(img2)
 	assert.NoError(t, err, "file on staging")
 	_, err = os.Stat(img3)

backend/app/store/image/image_test.go

@@ -17,7 +17,7 @@ import (
 func TestService_ExtractPictures(t *testing.T) {
 	svc := Service{ImageAPI: "/blah/"}
 	html := `blah <img src="/blah/user1/pic1.png"/> foo 
-<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
 	ids, err := svc.ExtractPictures(html)
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(ids), "two images")
@@ -25,6 +25,16 @@ func TestService_ExtractPictures(t *testing.T) {
 	assert.Equal(t, "user2/pic3.png", ids[1])
 }
 
+func TestService_ExtractPictures2(t *testing.T) {
+	svc := Service{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}
+	html := "<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." +
+		"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
+	ids, err := svc.ExtractPictures(html)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(ids), "one image in")
+	assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
+}
+
 func TestService_Cleanup(t *testing.T) {
 	store := MockStore{}
 	store.On("Cleanup", mock.Anything, mock.Anything).Times(10).Return(nil)

backend/app/store/service/service.go

@@ -1,3 +1,6 @@
+// Package service wraps engine interfaces with common logic unrelated to any particular engine implementation.
+// All consumers should be using service.DataStore and not the naked engine!
+
 package service
 
 import (
@@ -9,19 +12,19 @@ import (
 
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
-	cache "github.com/patrickmn/go-cache"
+	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
-	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // DataStore wraps store.Interface with additional methods
 type DataStore struct {
-	engine.Interface
+	Engine                 engine.Interface
 	EditDuration           time.Duration
 	AdminStore             admin.Store
 	MaxCommentSize         int
@@ -61,11 +64,13 @@ type PostMetaData struct {
 }
 
 const defaultCommentMaxSize = 2000
-const maxLastCommentsReply = 1000
+const maxLastCommentsReply = 5000
 
 // UnlimitedVotes doesn't restrict MaxVotes
 const UnlimitedVotes = -1
 
+var nonAdminUser = store.User{}
+
 // ErrRestrictedWordsFound returned in case comment text contains restricted words
 var ErrRestrictedWordsFound = errors.New("comment contains restricted words")
 
@@ -93,7 +98,56 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 	}()
 
 	s.submitImages(comment)
-	return s.Interface.Create(comment)
+	return s.Engine.Create(comment)
+}
+
+// Find wraps engine's Find call and alter results if needed. User used to alter comments
+// in order to differentiate between user's comments vs others comments.
+func (s *DataStore) Find(locator store.Locator, sort string, user store.User) ([]store.Comment, error) {
+	return s.FindSince(locator, sort, user, time.Time{})
+}
+
+// FindSince wraps engine's Find call and alter results if needed. Returns comments after since tx
+func (s *DataStore) FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error) {
+	req := engine.FindRequest{Locator: locator, Sort: sort, Since: since}
+	comments, err := s.Engine.Find(req)
+	if err != nil {
+		return comments, err
+	}
+
+	changedSort := false
+	// set votes controversy for comments added prior to #274
+	for i, c := range comments {
+		if c.Controversy == 0 && len(c.Votes) > 0 {
+			c.Controversy = s.controversy(s.upsAndDowns(c))
+			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+				changedSort = true
+			}
+		}
+		comments[i] = s.alterComment(c, user)
+	}
+
+	// resort commits if altered
+	if changedSort {
+		comments = engine.SortComments(comments, sort)
+	}
+
+	return comments, nil
+}
+
+// Get comment by ID
+func (s *DataStore) Get(locator store.Locator, commentID string, user store.User) (store.Comment, error) {
+	c, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
+	if err != nil {
+		return store.Comment{}, err
+	}
+	return s.alterComment(c, user), nil
+}
+
+// Put updates comment, mutable parts only
+func (s *DataStore) Put(locator store.Locator, comment store.Comment) error {
+	comment.Locator = locator
+	return s.Engine.Update(comment)
 }
 
 // submitImages initiated delayed commit of all images from the comment uploaded to remark42
@@ -101,7 +155,8 @@ func (s *DataStore) submitImages(comment store.Comment) {
 
 	s.ImageService.Submit(func() []string {
 		c := comment
-		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		// this can be called after last edit, we have to retrieve fresh comment
+		cc, err := s.Engine.Get(engine.GetRequest{Locator: c.Locator, CommentID: c.ID})
 		if err != nil {
 			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
 			return nil
@@ -141,14 +196,21 @@ func (s *DataStore) prepareNewComment(comment store.Comment) (store.Comment, err
 	return comment, nil
 }
 
+// DeleteAll removes all data from site
+func (s *DataStore) DeleteAll(siteID string) error {
+	req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}}
+	return s.Engine.Delete(req)
+}
+
 // SetPin pin/un-pin comment as special
 func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
-	comment, err := s.Get(locator, commentID)
+	comment, err := s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return err
 	}
 	comment.Pin = status
-	return s.Put(locator, comment)
+	comment.Locator = locator
+	return s.Engine.Update(comment)
 }
 
 // Vote for comment by id and locator
@@ -158,7 +220,7 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -207,8 +269,8 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 	}
 
 	comment.Vote = 0
-	if v, ok := comment.Votes[userID]; ok {
+	if vv, ok := comment.Votes[userID]; ok {
-		if v {
+		if vv {
 			comment.Vote = 1
 		} else {
 			comment.Vote = -1
@@ -216,8 +278,8 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 	}
 
 	comment.Controversy = s.controversy(s.upsAndDowns(comment))
-
+	comment.Locator = locator
-	return comment, s.Put(locator, comment)
+	return comment, s.Engine.Update(comment)
 }
 
 // controversy calculates controversial index of votes
@@ -246,7 +308,7 @@ type EditRequest struct {
 
 // EditComment to edit text and update Edit info
 func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -262,7 +324,8 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 
 	if req.Delete { // delete request
 		comment.Deleted = true
-		return comment, s.Delete(locator, commentID, store.SoftDelete)
+		delReq := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: store.SoftDelete}
+		return comment, s.Engine.Delete(delReq)
 	}
 
 	if s.RestrictedWordsMatcher != nil && s.RestrictedWordsMatcher.Match(comment.Locator.SiteID, req.Text) {
@@ -275,9 +338,10 @@ func (s *DataStore) EditComment(locator store.Locator, commentID string, req Edi
 		Timestamp: time.Now(),
 		Summary:   req.Summary,
 	}
-
+	comment.Locator = locator
 	comment.Sanitize()
-	err = s.Put(locator, comment)
+
+	err = s.Engine.Update(comment)
 	return comment, err
 }
 
@@ -295,7 +359,8 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 		return true
 	}
 
-	comments, err := s.Last(comment.Locator.SiteID, maxLastCommentsReply)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: comment.Locator.SiteID}, Limit: maxLastCommentsReply}
+	comments, err := s.Engine.Find(req)
 	if err != nil {
 		log.Printf("[WARN] can't get last comments for reply check, %v", err)
 		return false
@@ -312,13 +377,49 @@ func (s *DataStore) HasReplies(comment store.Comment) bool {
 	return false
 }
 
+// UserReplies returns list of all comments replied to given user
+func (s *DataStore) UserReplies(siteID, userID string, limit int, duration time.Duration) ([]store.Comment, string, error) {
+
+	comments, e := s.Last(siteID, maxLastCommentsReply, time.Time{}, nonAdminUser)
+	if e != nil {
+		return nil, "", errors.Wrap(e, "can't get last comments")
+	}
+	replies := []store.Comment{}
+
+	// get a comment for given userID in order to retrieve name
+	userName := ""
+	if cc, err := s.User(siteID, userID, 1, 0, nonAdminUser); err == nil && len(cc) > 0 {
+		userName = cc[0].User.Name
+	}
+
+	// collect replies
+	for _, c := range comments {
+
+		if len(replies) > limit || time.Since(c.Timestamp) > duration {
+			break
+		}
+
+		if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
+			var pc store.Comment
+			if pc, e = s.Get(c.Locator, c.ParentID, nonAdminUser); e != nil {
+				return nil, "", errors.Wrap(e, "can't get parent comment")
+			}
+			if pc.User.ID == userID {
+				replies = append(replies, c)
+			}
+		}
+	}
+
+	return replies, userName, nil
+}
+
 // SetTitle puts title from the locator.URL page and overwrites any existing title
 func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment store.Comment, err error) {
 	if s.TitleExtractor == nil {
 		return comment, errors.New("no title extractor")
 	}
 
-	comment, err = s.Get(locator, commentID)
+	comment, err = s.Engine.Get(engine.GetRequest{Locator: locator, CommentID: commentID})
 	if err != nil {
 		return comment, err
 	}
@@ -329,7 +430,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
 		return comment, err
 	}
 	comment.PostTitle = title
-	err = s.Put(locator, comment)
+	comment.Locator = locator
+	err = s.Engine.Update(comment)
 	return comment, err
 }
 
@@ -337,7 +439,8 @@ func (s *DataStore) SetTitle(locator store.Locator, commentID string) (comment s
 func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
 	res := []store.PostInfo{}
 	for _, p := range postIDs {
-		if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
+		req := engine.FindRequest{Locator: store.Locator{SiteID: siteID, URL: p}}
+		if c, err := s.Engine.Count(req); err == nil {
 			res = append(res, store.PostInfo{URL: p, Count: c})
 		}
 	}
@@ -364,7 +467,11 @@ func (s *DataStore) ValidateComment(c *store.Comment) error {
 
 // IsAdmin checks if usesID in the list of admins
 func (s *DataStore) IsAdmin(siteID string, userID string) bool {
-	for _, a := range s.AdminStore.Admins(siteID) {
+	admins, err := s.AdminStore.Admins(siteID)
+	if err != nil {
+		return false
+	}
+	for _, a := range admins {
 		if a == userID {
 			return true
 		}
@@ -372,20 +479,126 @@ func (s *DataStore) IsAdmin(siteID string, userID string) bool {
 	return false
 }
 
+// IsReadOnly checks if post read-only
+func (s *DataStore) IsReadOnly(locator store.Locator) bool {
+	req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetReadOnly set/reset read-only flag
+func (s *DataStore) SetReadOnly(locator store.Locator, status bool) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+
+	}
+	req := engine.FlagRequest{Locator: locator, Flag: engine.ReadOnly, Update: roStatus}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// IsVerified checks if user verified
+func (s *DataStore) IsVerified(siteID string, userID string) bool {
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetVerified set/reset verified status for user
+func (s *DataStore) SetVerified(siteID string, userID string, status bool) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+	}
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Verified, Update: roStatus}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// IsBlocked checks if user blocked
+func (s *DataStore) IsBlocked(siteID string, userID string) bool {
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Flag: engine.Blocked}
+	ro, err := s.Engine.Flag(req)
+	return err == nil && ro
+}
+
+// SetBlock set/reset verified status for user
+func (s *DataStore) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
+	roStatus := engine.FlagFalse
+	if status {
+		roStatus = engine.FlagTrue
+	}
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID,
+		Flag: engine.Blocked, Update: roStatus, TTL: ttl}
+	_, err := s.Engine.Flag(req)
+	return err
+}
+
+// Blocked returns list with all blocked users
+func (s *DataStore) Blocked(siteID string) (res []store.BlockedUser, err error) {
+	blocked, e := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Blocked})
+	if e != nil {
+		return nil, errors.Wrapf(err, "can't get list of blocked users for %s", siteID)
+	}
+	for _, v := range blocked {
+		res = append(res, v.(store.BlockedUser))
+	}
+	return res, nil
+}
+
+// Info get post info
+func (s *DataStore) Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) {
+	req := engine.InfoRequest{Locator: locator, ReadOnlyAge: readonlyAge}
+	res, err := s.Engine.Info(req)
+	if err != nil {
+		return store.PostInfo{}, err
+	}
+	if len(res) == 0 {
+		return store.PostInfo{}, errors.Errorf("post %+v not found", locator)
+	}
+	return res[0], nil
+}
+
+// Delete comment by id
+func (s *DataStore) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
+	req := engine.DeleteRequest{Locator: locator, CommentID: commentID, DeleteMode: mode}
+	return s.Engine.Delete(req)
+}
+
+// DeleteUser removes all comments from user
+func (s *DataStore) DeleteUser(siteID string, userID string, mode store.DeleteMode) error {
+	req := engine.DeleteRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, DeleteMode: mode}
+	return s.Engine.Delete(req)
+}
+
+// List of commented posts
+func (s *DataStore) List(siteID string, limit int, skip int) ([]store.PostInfo, error) {
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Skip: skip}
+	return s.Engine.Info(req)
+}
+
+// Count gets number of comments for the post
+func (s *DataStore) Count(locator store.Locator) (int, error) {
+	req := engine.FindRequest{Locator: locator}
+	return s.Engine.Count(req)
+}
+
 // Metas returns metadata for users and posts
 func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMetaData, err error) {
 	umetas = []UserMetaData{}
 	pmetas = []PostMetaData{}
+
 	// set posts meta
-	posts, err := s.List(siteID, 0, 0)
+	posts, err := s.Engine.Info(engine.InfoRequest{Locator: store.Locator{SiteID: siteID}})
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "can't get list of posts for %s", siteID)
 	}
+
 	for _, p := range posts {
 		if s.IsReadOnly(store.Locator{SiteID: siteID, URL: p.URL}) {
 			pmetas = append(pmetas, PostMetaData{URL: p.URL, ReadOnly: true})
 		}
-
 	}
 
 	// set users meta
@@ -407,11 +620,12 @@ func (s *DataStore) Metas(siteID string) (umetas []UserMetaData, pmetas []PostMe
 	}
 
 	// process verified users
-	verified, err := s.Verified(siteID)
+	verified, err := s.Engine.ListFlags(engine.FlagRequest{Locator: store.Locator{SiteID: siteID}, Flag: engine.Verified})
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "can't get list of verified users for %s", siteID)
 	}
-	for _, v := range verified {
+	for _, vi := range verified {
+		v := vi.(string)
 		val, ok := m[v]
 		if !ok {
 			val = UserMetaData{ID: v}
@@ -452,30 +666,35 @@ func (s *DataStore) SetMetas(siteID string, umetas []UserMetaData, pmetas []Post
 	return errs.ErrorOrNil()
 }
 
-// Find wraps engine's Find call and alter results if needed
+// User gets comment for given userID on siteID
-func (s *DataStore) Find(locator store.Locator, sort string) ([]store.Comment, error) {
+func (s *DataStore) User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error) {
-	comments, err := s.Interface.Find(locator, sort)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID, Limit: limit, Skip: skip}
+	comments, err := s.Engine.Find(req)
 	if err != nil {
 		return comments, err
 	}
+	return s.alterComments(comments, user), nil
+}
 
-	changedSort := false
+// UserCount is comments count by user
-	// set votes controversy for comments added prior to #274
+func (s *DataStore) UserCount(siteID, userID string) (int, error) {
-	for i, c := range comments {
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, UserID: userID}
-		if c.Controversy == 0 && len(c.Votes) > 0 {
+	return s.Engine.Count(req)
-			comments[i].Controversy = s.controversy(s.upsAndDowns(c))
+}
-			if !changedSort && strings.Contains(sort, "controversy") { // trigger sort change
+
-				changedSort = true
+// Last gets last comments for site, cross-post. Limited by count and optional since ts
-			}
+func (s *DataStore) Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error) {
-		}
+	req := engine.FindRequest{Locator: store.Locator{SiteID: siteID}, Limit: limit, Since: since, Sort: "-time"}
+	comments, err := s.Engine.Find(req)
+	if err != nil {
+		return comments, err
 	}
+	return s.alterComments(comments, user), nil
+}
 
-	// resort commits if altered
+// Close store service
-	if changedSort {
+func (s *DataStore) Close() error {
-		comments = engine.SortComments(comments, sort)
+	return s.Engine.Close()
-	}
-
-	return comments, nil
 }
 
 func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
@@ -503,3 +722,53 @@ func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 
 	return lock
 }
+
+func (s *DataStore) alterComments(cc []store.Comment, user store.User) (res []store.Comment) {
+	res = make([]store.Comment, len(cc))
+	for i, c := range cc {
+		res[i] = s.alterComment(c, user)
+	}
+	return res
+}
+
+func (s *DataStore) alterComment(c store.Comment, user store.User) (res store.Comment) {
+
+	blocReq := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+	blocked, bErr := s.Engine.Flag(blocReq)
+
+	// mark user blocked
+	if bErr == nil && blocked {
+		c.User.Blocked = blocked
+	}
+
+	// set verified status retroactively
+	if !c.User.Blocked {
+		verifReq := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: c.Locator.SiteID}, UserID: c.User.ID}
+		c.User.Verified, _ = s.Engine.Flag(verifReq)
+	}
+
+	// hide info from non-admins
+	if !user.Admin {
+		c.User.IP = ""
+	}
+
+	c = s.prepVotes(c, user)
+	return c
+}
+
+// prepare vote info for client view
+func (s *DataStore) prepVotes(c store.Comment, user store.User) store.Comment {
+
+	c.Vote = 0 // default is "none" (not voted)
+
+	if v, ok := c.Votes[user.ID]; ok {
+		if v {
+			c.Vote = 1
+		} else {
+			c.Vote = -1
+		}
+	}
+
+	c.Votes = nil // hide voters list
+	return c
+}

backend/app/store/service/service_test.go

@@ -18,11 +18,11 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 var testDb = "/tmp/test-remark.db"
@@ -30,7 +30,7 @@ var testDb = "/tmp/test-remark.db"
 func TestService_CreateFromEmpty(t *testing.T) {
 	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
 		Text:    "text",
 		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
@@ -40,7 +40,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, id != "", id)
 
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
+	res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id))
 	assert.NoError(t, err)
 	t.Logf("%+v", res)
 	assert.Equal(t, "text", res.Text)
@@ -54,7 +54,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
 func TestService_CreateFromPartial(t *testing.T) {
 	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
 		Text:      "text",
 		Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
@@ -66,7 +66,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, id != "", id)
 
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
+	res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id))
 	assert.NoError(t, err)
 	t.Logf("%+v", res)
 	assert.Equal(t, "text", res.Text)
@@ -81,7 +81,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
 	comment := store.Comment{
 		Text:      "text",
@@ -94,15 +94,15 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, id != "", id)
 
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
+	res, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id))
 	assert.NoError(t, err)
 	t.Logf("%+v", res)
-	assert.Equal(t, "Радио-Т 630", res.PostTitle)
+	assert.Equal(t, "Радио-Т 630 — Радио-Т Подкаст", res.PostTitle)
 
 	comment.PostTitle = "post blah"
 	id, err = b.Create(comment)
 	assert.NoError(t, err)
-	res, err = b.Get(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id)
+	res, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com/p/2018/12/29/podcast-630/", SiteID: "radio-t"}, id))
 	assert.NoError(t, err)
 	t.Logf("%+v", res)
 	assert.Equal(t, "post blah", res.PostTitle, "keep comment title")
@@ -131,7 +131,7 @@ func TestService_SetTitle(t *testing.T) {
 	defer tss.Close()
 
 	ks := admin.NewStaticKeyStore("secret 123")
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
 	comment := store.Comment{
 		Text:      "text",
@@ -145,7 +145,7 @@ func TestService_SetTitle(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, id != "", id)
 
-	res, err := b.Get(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
+	res, err := b.Engine.Get(getReq(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id))
 	assert.NoError(t, err)
 	t.Logf("%+v", res)
 	assert.Equal(t, "", res.PostTitle)
@@ -157,14 +157,14 @@ func TestService_SetTitle(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "post1 blah 123", c.PostTitle)
 
-	b = DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
+	b = DataStore{Engine: prepStoreEngine(t), AdminStore: ks}
 	_, err = b.SetTitle(store.Locator{URL: tss.URL + "/post1", SiteID: "radio-t"}, id)
 	require.EqualError(t, err, "no title extractor")
 }
 
 func TestService_Vote(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
 		Text:    "text",
@@ -174,19 +174,32 @@ func TestService_Vote(t *testing.T) {
 	_, err := b.Create(comment)
 	assert.NoError(t, err)
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
 	assert.Equal(t, 0, res[0].Vote)
 	assert.Equal(t, map[string]bool(nil), res[0].Votes, "no votes initially")
 
+	// vote +1 as user1
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 1, c.Score)
 	assert.Equal(t, 1, c.Vote)
 	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
+	// check result as user1
+	c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user1"})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c.Score)
+	assert.Equal(t, 1, c.Vote, "can see own vote result")
+	assert.Nil(t, c.Votes)
+	// check result as user2
+	c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.User{ID: "user2"})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c.Score)
+	assert.Equal(t, 0, c.Vote, "can't see other user vote result")
+	assert.Nil(t, c.Votes)
 
 	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
 	assert.NotNil(t, err, "self-voting not allowed")
@@ -195,17 +208,28 @@ func TestService_Vote(t *testing.T) {
 	assert.NotNil(t, err, "double-voting rejected")
 	assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted"))
 
-	res, err = b.Last("radio-t", 0)
+	// check in last as user1
-	assert.Nil(t, err)
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"})
+	assert.NoError(t, err)
+	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 1, res[0].Score)
 	assert.Equal(t, 1, res[0].Vote)
 	assert.Equal(t, 0.0, res[0].Controversy)
 
+	// check in last as user2
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user2"})
+	assert.NoError(t, err)
+	t.Logf("%+v", res[0])
+	assert.Equal(t, 3, len(res))
+	assert.Equal(t, 1, res[0].Score)
+	assert.Equal(t, 0, res[0].Vote)
+	assert.Equal(t, 0.0, res[0].Controversy)
+
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
-	assert.Nil(t, err, "vote reset")
+	assert.NoError(t, err, "vote reset")
-	res, err = b.Last("radio-t", 0)
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
 	assert.Equal(t, 0, res[0].Vote)
@@ -214,25 +238,25 @@ func TestService_Vote(t *testing.T) {
 
 func TestService_VoteLimit(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user4", true)
 	assert.NotNil(t, err, "vote limit reached")
 	assert.True(t, strings.HasPrefix(err.Error(), "maximum number of votes exceeded for comment id-1"))
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user4", true)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 }
 
 func TestService_VotesDisabled(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
 	assert.EqualError(t, err, "maximum number of votes exceeded for comment id-1")
@@ -240,7 +264,7 @@ func TestService_VotesDisabled(t *testing.T) {
 
 func TestService_VoteAggressive(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
 		Text:    "text",
@@ -250,8 +274,8 @@ func TestService_VoteAggressive(t *testing.T) {
 	_, err := b.Create(comment)
 	assert.NoError(t, err)
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 0, res[0].Score)
@@ -259,7 +283,7 @@ func TestService_VoteAggressive(t *testing.T) {
 
 	// add a vote as user2
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	// crazy vote +1 as user1
 	var wg sync.WaitGroup
@@ -271,13 +295,14 @@ func TestService_VoteAggressive(t *testing.T) {
 		}()
 	}
 	wg.Wait()
-	res, err = b.Last("radio-t", 0)
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{ID: "user1"})
 	require.NoError(t, err)
 
 	t.Logf("%+v", res[0])
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, 2, res[0].Score, "add single +1")
-	assert.Equal(t, 2, len(res[0].Votes), "made a single vote")
+	assert.Equal(t, 1, res[0].Vote, "user1 voted +1")
+	assert.Equal(t, 0, len(res[0].Votes), "votes hidden")
 
 	// random +1/-1 result should be [0..2]
 	rand.Seed(time.Now().UnixNano())
@@ -290,7 +315,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		}()
 	}
 	wg.Wait()
-	res, err = b.Last("radio-t", 0)
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
 	require.NoError(t, err)
 	assert.Equal(t, 3, len(res))
 	t.Logf("%+v %d", res[0], res[0].Score)
@@ -300,7 +325,7 @@ func TestService_VoteAggressive(t *testing.T) {
 func TestService_VoteConcurrent(t *testing.T) {
 
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
 		Text:    "text",
@@ -309,7 +334,7 @@ func TestService_VoteConcurrent(t *testing.T) {
 	}
 	_, err := b.Create(comment)
 	assert.NoError(t, err)
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	require.Nil(t, err)
 
 	// concurrent vote +1 as multiple users for the same comment
@@ -324,35 +349,35 @@ func TestService_VoteConcurrent(t *testing.T) {
 		}()
 	}
 	wg.Wait()
-	res, err = b.Last("radio-t", 0)
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
 	require.NoError(t, err)
 	assert.Equal(t, 100, res[0].Score, "should have 100 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 100 votes")
+	assert.Equal(t, 0, len(res[0].Votes), "should hide votes")
 	assert.Equal(t, 0.0, res[0].Controversy, "should have 0 controversy")
 }
 
 func TestService_VotePositive(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
 		MaxVotes: -1, PositiveScore: true}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
 	assert.EqualError(t, err, "minimal score reached for comment id-1")
 
 	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user3", true)
-	assert.Nil(t, err, "minimal score doesn't affect positive vote")
+	assert.NoError(t, err, "minimal score doesn't affect positive vote")
 
-	b = DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
+	b = DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
 		MaxVotes: -1, PositiveScore: false}
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", false)
-	assert.Nil(t, err, "minimal score ignored")
+	assert.NoError(t, err, "minimal score ignored")
 	assert.Equal(t, -1, c.Score)
 	assert.Equal(t, 0.0, c.Controversy)
 }
 
 func TestService_VoteControversy(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
 	assert.NoError(t, err)
@@ -370,7 +395,7 @@ func TestService_VoteControversy(t *testing.T) {
 	assert.InDelta(t, 1.73, c.Controversy, 0.01)
 
 	// check if stored
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	require.NoError(t, err)
 	assert.Equal(t, 1, res[0].Score, "should have 1 score")
 	assert.InDelta(t, 1.73, res[0].Controversy, 0.01)
@@ -402,81 +427,82 @@ func TestService_Controversy(t *testing.T) {
 
 func TestService_Pin(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 	assert.Equal(t, false, res[0].Pin)
 
 	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
+	c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, true, c.Pin)
 
 	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
-	c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
+	c, err = b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, false, c.Pin)
 }
 
 func TestService_EditComment(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 	assert.Nil(t, res[0].Edit)
 
 	comment, err := b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
 		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "my edit", comment.Edit.Summary)
 	assert.Equal(t, "xxx", comment.Text)
 	assert.Equal(t, "yyy", comment.Orig)
 
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
+	c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, "my edit", c.Edit.Summary)
 	assert.Equal(t, "xxx", c.Text)
 
 	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
 		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err, "allow second edit")
+	assert.NoError(t, err, "allow second edit")
 }
 
 func TestService_DeleteComment(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 	assert.Nil(t, res[0].Edit)
 
 	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, EditRequest{Delete: true})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
+	c, err := b.Engine.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, c.Deleted)
 	t.Logf("%+v", c)
 }
 
 func TestService_EditCommentDurationFailed(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123")}
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 	assert.Nil(t, res[0].Edit)
 
@@ -489,11 +515,11 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
 
 func TestService_EditCommentReplyFailed(t *testing.T) {
 	defer teardown(t)
-	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
+	b := DataStore{Engine: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
-	res, err := b.Last("radio-t", 0)
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
 	t.Logf("%+v", res[1])
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 	assert.Nil(t, res[1].Edit)
 
@@ -531,9 +557,10 @@ func TestService_ValidateComment(t *testing.T) {
 	for n, tt := range tbl {
 		e := b.ValidateComment(&tt.inp)
 		if tt.err == nil {
-			assert.Nil(t, e, "check #%d", n)
+			assert.NoError(t, e, "check #%d", n)
 			continue
 		}
+		require.NotNil(t, e)
 		assert.EqualError(t, tt.err, e.Error(), "check #%d", n)
 	}
 }
@@ -551,15 +578,15 @@ func TestService_Counts(t *testing.T) {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err := b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
-	svc := DataStore{Interface: b}
+	svc := DataStore{Engine: b}
 	res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res)
 
 	res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, []store.PostInfo{
 		{URL: "https://radio-t.com", Count: 2},
 		{URL: "https://radio-t.com/2", Count: 1},
@@ -570,7 +597,7 @@ func TestService_Counts(t *testing.T) {
 func TestService_GetMetas(t *testing.T) {
 	defer teardown(t)
 	// two comments for https://radio-t.com
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	um, pm, err := b.Metas("radio-t")
@@ -601,7 +628,7 @@ func TestService_GetMetas(t *testing.T) {
 func TestService_SetMetas(t *testing.T) {
 	defer teardown(t)
 	// two comments for https://radio-t.com
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
 	umetas := []UserMetaData{}
 	pmetas := []PostMetaData{}
@@ -625,7 +652,7 @@ func TestService_SetMetas(t *testing.T) {
 func TestService_IsAdmin(t *testing.T) {
 	defer teardown(t)
 	// two comments for https://radio-t.com
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
 
 	assert.False(t, b.IsAdmin("radio-t", "user1"))
@@ -636,7 +663,7 @@ func TestService_HasReplies(t *testing.T) {
 	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
 
 	comment := store.Comment{
@@ -662,14 +689,93 @@ func TestService_HasReplies(t *testing.T) {
 	assert.True(t, b.HasReplies(comment))
 }
 
+func TestService_UserReplies(t *testing.T) {
+
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t),
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	c1 := store.Comment{
+		ID:      "comment-id-1",
+		Text:    "test 123",
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one u1"},
+	}
+	c2 := store.Comment{
+		ID:       "comment-id-2",
+		ParentID: "comment-id-1",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u2", Name: "developer one u2"},
+	}
+	c3 := store.Comment{
+		ID:       "comment-id-3",
+		ParentID: "comment-id-1",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u2", Name: "developer one u3"},
+	}
+	c4 := store.Comment{
+		ID:       "comment-id-4",
+		ParentID: "",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u4", Name: "developer one u4"},
+	}
+
+	c5 := store.Comment{
+		ID:       "comment-id-5",
+		ParentID: "comment-id-1",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u2", Name: "developer one u2"},
+	}
+
+	_, err := b.Create(c1)
+	require.NoError(t, err)
+	_, err = b.Create(c2)
+	require.NoError(t, err)
+	_, err = b.Create(c3)
+	require.NoError(t, err)
+	_, err = b.Create(c4)
+	require.NoError(t, err)
+
+	time.Sleep(100 * time.Millisecond)
+	_, err = b.Create(c5)
+	require.NoError(t, err)
+
+	cc, u, err := b.UserReplies("radio-t", "u1", 10, time.Hour)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(cc), "3 replies to u1")
+	assert.Equal(t, "developer one u1", u)
+
+	cc, u, err = b.UserReplies("radio-t", "u1", 10, time.Millisecond*50)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(cc), "1 reply to u1 in last 90ms")
+	assert.Equal(t, "developer one u1", u)
+
+	cc, u, err = b.UserReplies("radio-t", "u2", 10, time.Hour)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(cc), "0 replies to u2")
+	assert.Equal(t, "developer one u2", u)
+
+	cc, u, err = b.UserReplies("radio-t", "uxxx", 10, time.Hour)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(cc), "0 replies to uxxx")
+	assert.Equal(t, "", u)
+
+}
+
 func TestService_Find(t *testing.T) {
 	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
 
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{})
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(res))
 
@@ -683,11 +789,11 @@ func TestService_Find(t *testing.T) {
 		Score:     1,
 		Votes:     map[string]bool{"id-1": true, "id-2": true, "123456": false},
 	}
-	_, err = b.Interface.Create(comment) // create directly with engine, doesn't set Controversy
+	_, err = b.Engine.Create(comment) // create directly with engine, doesn't set Controversy
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	// make sure Controversy altered
-	res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy")
+	res, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "-controversy", store.User{})
 	require.NoError(t, err)
 	assert.Equal(t, 3, len(res))
 	assert.Equal(t, "123456", res[0].ID)
@@ -696,6 +802,209 @@ func TestService_Find(t *testing.T) {
 	assert.InDelta(t, 0, res[1].Controversy, 0.01)
 }
 
+func TestService_FindSince(t *testing.T) {
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	res, err := b.FindSince(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{}, time.Time{})
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(res))
+	assert.Equal(t, "id-1", res[0].ID)
+
+	res, err = b.FindSince(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time", store.User{},
+		time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local))
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+	assert.Equal(t, "id-2", res[0].ID)
+}
+
+func TestService_Info(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	info, err := b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
+	require.NoError(t, err)
+	assert.Equal(t, "https://radio-t.com", info.URL)
+	assert.Equal(t, 2, info.Count)
+	assert.False(t, info.ReadOnly)
+	assert.True(t, info.LastTS.After(info.FirstTS))
+
+	time.Sleep(1 * time.Second) // make post RO in 1sec
+	info, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 1)
+	require.NoError(t, err)
+	assert.Equal(t, "https://radio-t.com", info.URL)
+	assert.True(t, info.ReadOnly)
+}
+
+func TestService_Delete(t *testing.T) {
+	defer teardown(t)
+
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+	assert.Equal(t, 2, len(res))
+	assert.NoError(t, err)
+
+	err = b.Delete(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, store.SoftDelete)
+	assert.NoError(t, err)
+
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
+	assert.Equal(t, 1, len(res), "one left")
+	assert.NoError(t, err)
+}
+
+// DeleteUser removes all comments from user
+func TestService_DeleteUser(t *testing.T) {
+	defer teardown(t)
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	// add one more for user2
+	comment := store.Comment{
+		ID:        "123456xyz",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+	assert.Equal(t, 3, len(res), "3 comments initially, for 2 diff users and 2 posts")
+	assert.NoError(t, err)
+
+	err = b.DeleteUser("radio-t", "user1", store.HardDelete)
+	assert.NoError(t, err)
+
+	res, err = b.Last("radio-t", 0, time.Time{}, store.User{})
+	assert.Equal(t, 1, len(res), "only one comment left for user2")
+	assert.NoError(t, err)
+	assert.Equal(t, "user2", res[0].User.ID)
+}
+
+func TestService_List(t *testing.T) {
+	defer teardown(t)
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	// add one more for user2
+	comment := store.Comment{
+		ID:        "id-3",
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	res, err := b.List("radio-t", 0, 0)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "2 posts")
+	assert.Equal(t, "https://radio-t.com/2", res[0].URL)
+	assert.Equal(t, 1, res[0].Count)
+	assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].FirstTS)
+	assert.Equal(t, time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local), res[0].LastTS)
+
+	assert.Equal(t, "https://radio-t.com", res[1].URL)
+	assert.Equal(t, 2, res[1].Count)
+	assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local), res[1].FirstTS)
+	assert.Equal(t, time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local), res[1].LastTS)
+}
+
+func TestService_Count(t *testing.T) {
+	defer teardown(t)
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	// add one more for user2
+	comment := store.Comment{
+		ID:        "id-3",
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	c, err = b.Count(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c)
+
+	c, err = b.Count(store.Locator{URL: "https://radio-t.com/3", SiteID: "radio-t"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestService_UserCount(t *testing.T) {
+	defer teardown(t)
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	// add one more for user2
+	comment := store.Comment{
+		ID:        "id-3",
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	c, err := b.UserCount("radio-t", "user1")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, c)
+
+	c, err = b.UserCount("radio-t", "user2")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, c)
+
+	_, err = b.UserCount("radio-t", "userBad")
+	assert.EqualError(t, err, "no comments for user userBad in store for radio-t site")
+}
+
+func TestService_DeleteAll(t *testing.T) {
+	defer teardown(t)
+	// two comments for https://radio-t.com, no reply
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
+		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
+
+	// add one more for user2
+	comment := store.Comment{
+		ID:        "id-3",
+		Timestamp: time.Date(2018, 12, 20, 15, 18, 22, 0, time.Local),
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user2", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	err = b.DeleteAll("radio-t")
+	assert.NoError(t, err)
+
+	res, err := b.Last("radio-t", 0, time.Time{}, store.User{})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
 func TestService_submitImages(t *testing.T) {
 	defer teardown(t)
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
@@ -705,7 +1014,7 @@ func TestService_submitImages(t *testing.T) {
 	imgSvc := &image.Service{Store: &mockStore, TTL: time.Millisecond * 50}
 
 	// two comments for https://radio-t.com
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+	b := DataStore{Engine: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
 
 	c := store.Comment{
@@ -715,19 +1024,52 @@ func TestService_submitImages(t *testing.T) {
 		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
-	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	_, err := b.Engine.Create(c) // create directly with engine, doesn't call submitImages
 	assert.NoError(t, err)
 
 	b.submitImages(c)
 	time.Sleep(250 * time.Millisecond)
 }
 
+func TestService_alterComment(t *testing.T) {
+	defer teardown(t)
+
+	engineMock := engine.MockInterface{}
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil)
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
+	svc := DataStore{Engine: &engineMock}
+
+	r := svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
+		store.User{Name: "dev", ID: "devid", Admin: false})
+	assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid"}}, r, "ip cleaned")
+	r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}},
+		store.User{Name: "dev", ID: "devid", Admin: true})
+	assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid"}}, r, "ip not cleaned")
+
+	engineMock = engine.MockInterface{}
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(false, nil)
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(true, nil)
+	svc = DataStore{Engine: &engineMock}
+	r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}},
+		store.User{Name: "dev", ID: "devid", Admin: false})
+	assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", ID: "devid", Verified: true}}, r, "verified set")
+
+	engineMock = engine.MockInterface{}
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Blocked, UserID: "devid"}).Return(true, nil)
+	engineMock.On("Flag", engine.FlagRequest{Flag: engine.Verified, UserID: "devid"}).Return(false, nil)
+	svc = DataStore{Engine: &engineMock}
+	r = svc.alterComment(store.Comment{ID: "123", User: store.User{IP: "127.0.0.1", ID: "devid", Verified: true}},
+		store.User{Name: "dev", ID: "devid", Admin: false})
+	assert.Equal(t, store.Comment{ID: "123", User: store.User{IP: "", Verified: true, Blocked: true, ID: "devid"},
+		Deleted: false}, r, "blocked")
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
 	_ = os.Remove(testDb)
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	b := boltStore
 
 	comment := store.Comment{
@@ -738,7 +1080,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	comment = store.Comment{
 		ID:        "id-2",
@@ -748,7 +1090,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 		User:      store.User{ID: "user1", Name: "user name"},
 	}
 	_, err = b.Create(comment)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 
 	return b
 }
@@ -756,3 +1098,10 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 func teardown(_ *testing.T) {
 	_ = os.Remove(testDb)
 }
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/app/store/service/title.go

@@ -3,6 +3,7 @@ package service
 import (
 	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/go-pkgz/lcw"
@@ -44,7 +45,11 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
-		defer resp.Body.Close() //nolint
+		defer func() {
+			if err = resp.Body.Close(); err != nil {
+				log.Printf("[WARN] failed to close title extractor body, %v", err)
+			}
+		}()
 		if resp.StatusCode != 200 {
 			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
 		}
@@ -81,7 +86,10 @@ func (t *TitleExtractor) isTitleElement(n *html.Node) bool {
 
 func (t *TitleExtractor) traverse(n *html.Node) (string, bool) {
 	if t.isTitleElement(n) {
-		return n.FirstChild.Data, true
+		title := n.FirstChild.Data
+		title = strings.Replace(title, "\n", "", -1)
+		title = strings.TrimSpace(title)
+		return title, true
 	}
 
 	for c := n.FirstChild; c != nil; c = c.NextSibling {

backend/app/store/service/title_test.go

@@ -24,7 +24,8 @@ func TestTitle_GetTitle(t *testing.T) {
 		title string
 	}{
 		{`<html><title>blah 123</title><body> 2222</body></html>`, true, "blah 123"},
-		{`<html><title>blah 123 `, true, "blah 123 "},
+		{`<html><title>blah 123 `, true, "blah 123"},
+		{"<html><title>\n\n  blah 123 \n ", true, "blah 123"},
 		{`<html><body> 2222</body></html>`, false, ""},
 	}
 
@@ -44,7 +45,7 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			_, err := w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>\n\n blah 123\n</title><body> 2222</body></html>"))
 			assert.NoError(t, err)
 			return
 		}

backend/app/store/service/tree.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"sort"

backend/app/store/service/tree_test.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"encoding/json"

backend/app/store/user.go

@@ -2,7 +2,7 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1" //nolint
+	"crypto/sha1" // nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -41,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id) //nolint
+	return hashWithFallback(sha1.New(), id) // nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -2,40 +2,40 @@ module github.com/umputun/remark/backend
 
 go 1.12
 
+replace gopkg.in/russross/blackfriday.v2 => github.com/russross/blackfriday/v2 v2.0.1
+
 require (
-	github.com/PuerkitoBio/goquery v1.4.0
+	cloud.google.com/go v0.41.0 // indirect
-	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/PuerkitoBio/goquery v1.5.0
-	github.com/coreos/bbolt v1.3.0
+	github.com/coreos/bbolt v1.3.3
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/didip/tollbooth v4.0.0+incompatible
 	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
-	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
 	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/go-chi/cors v1.0.0
-	github.com/go-chi/render v1.0.0
+	github.com/go-chi/render v1.0.1
-	github.com/go-pkgz/auth v0.5.0
+	github.com/go-pkgz/auth v0.7.2
-	github.com/go-pkgz/lcw v0.2.0
+	github.com/go-pkgz/lcw v0.3.1
-	github.com/go-pkgz/lgr v0.6.2
+	github.com/go-pkgz/lgr v0.6.3
-	github.com/go-pkgz/mongo v1.1.2
+	github.com/go-pkgz/repeater v1.1.2
-	github.com/go-pkgz/repeater v1.1.1
+	github.com/go-pkgz/rest v1.4.1
-	github.com/go-pkgz/rest v1.4.0
+	github.com/go-pkgz/syncs v1.1.1
-	github.com/go-pkgz/syncs v1.1.0
+	github.com/google/uuid v1.1.1
-	github.com/golang/mock v1.2.0
+	github.com/gorilla/feeds v1.1.1
-	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
+	github.com/hashicorp/go-multierror v1.0.0
-	github.com/gorilla/feeds v1.1.0
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
 	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
-	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
+	github.com/kyokomi/emoji v2.1.0+incompatible
+	github.com/microcosm-cc/bluemonday v1.0.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.8.1
-	github.com/rakyll/statik v0.1.3
+	github.com/rakyll/statik v0.1.6
 	github.com/rs/xid v1.2.1
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.3.0
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b
+	golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9
-	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
-	gopkg.in/russross/blackfriday.v2 v2.0.0
+	gopkg.in/russross/blackfriday.v2 v2.0.1
 )

backend/go.sum

@@ -1,13 +1,19 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
+cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM=
+cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
-github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -24,48 +30,67 @@ github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAU
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
 github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
-github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
-github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-pkgz/auth v0.5.0 h1:+wqppq35x83PchZNZ7SHHYLI/e8WeETFouujDLsklac=
+github.com/go-pkgz/auth v0.7.2 h1:+LvAgqwQtYuWphpZE8qLtspVd65+VgreJkFLsKrtmmk=
-github.com/go-pkgz/auth v0.5.0/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/auth v0.7.2/go.mod h1:ibOpZYISiaOvAHe2bsKj2s3v4AkMam2WxxIFn+zhulo=
-github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
+github.com/go-pkgz/lcw v0.3.1 h1:PhfB0xNUawLMlx5rXvOTIc7d5LMrr1GM9vIzmG96aUI=
-github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lcw v0.3.1/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
 github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
 github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
-github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
+github.com/go-pkgz/lgr v0.6.3 h1:n9pGk2paBV8w/Y/FVEq5MkwDmP33dnUPKbY4CyyygwM=
+github.com/go-pkgz/lgr v0.6.3/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
 github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
-github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
+github.com/go-pkgz/repeater v1.1.2 h1:OxTyUMdEGiN4jRk5g3HHWQ6o4GDezjCLwpmDa/On+mU=
-github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/repeater v1.1.2/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
-github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
-github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
-github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.1 h1:jWN+y6FS/Xe+8z4l3QMbSnODGyaxDHGojIS+wyKIjxg=
-github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
-github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
-github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
-github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
+github.com/kyokomi/emoji v2.1.0+incompatible h1:+DYU2RgpI6OHG4oQkM5KlqD3Wd3UPEsX8jamTo1Mp6o=
-github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
 github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -75,43 +100,121 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
+github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
-github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff h1:+2zgJKVDVAz/BWSsuniCmU1kLCjL88Z8/kv39xCI9NQ=
+golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9 h1:uc17S921SPw5F2gJo7slQ3aqvr2RwpL7eb3+DZncu3s=
+golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/remark.rest

@@ -1,11 +1,11 @@
 
 ### find request with tree
-GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=tree&url={{url}}
+GET {{host}}/api/v1/find?site={{site}}&sort=-time&format=tree&url={{url}}
 
 ### find request with plain
 GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
 
-### find request with plain
+### find request with plain. dev token for secret=secret, not admin
 GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
 X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
@@ -104,7 +104,7 @@ GET {{host}}/api/v1/admin/blocked?site={{site}}
 DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
 
 ### get post info
-GET {{host}}/api/v1/info?site={{site}}&url=https://radio-t.com/p/2018/05/08/prep-597/
+GET {{host}}/api/v1/info?site={{site}}&url={{url}
 
 ### post rss
 GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
@@ -112,7 +112,7 @@ GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
 ### site rss
 GET {{host}}/api/v1/rss/site?site={{site}}
 
-### replie rss
+### reply rss
 GET {{host}}/api/v1/rss/reply?site={{site}}&user={{user}}
 
 ### get default avatar
@@ -121,5 +121,14 @@ GET {{host}}/api/v1/avatar/blah
 ### get config
 GET  {{host}}/api/v1/config?site={{site}}
 
+### deleteme (user's request). dev token for secret=secret, not admin
+POST {{host}}/api/v1/deleteme?site_id={{site}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
+
+### deletme, admin. admin token for secret=secret
+GET {{host}}/api/v1/admin/deleteme?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1NjU2NjI4MDYsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTU3NzEzOTQ2LCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiIiwiYXR0cnMiOnsiYWRtaW4iOmZhbHNlLCJibG9ja2VkIjpmYWxzZSwiZGVsZXRlX21lIjp0cnVlfX19.qmnsQt_jilHzoauA9D7t1m3w69qvAJsZkuKVPWhpdik
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE2NTc3MDQ5NzksImp0aSI6Ijk4ZDNhOGFkMGNjZWU5M2Q1MWYwYjJiOTY1ZjU2YmE2NmJkNmZiNzYiLCJpYXQiOjE1NTc3MDQ2NzksImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.YIt9Zq3n0O8PMkR78pGAqyHI0exCU0vLKjcHgcXfLFw
+
+
 ### ping
 GET {{host}}/ping

backend/vendor/cloud.google.com/go/AUTHORS

@@ -1,15 +0,0 @@
-# This is the official list of cloud authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as:
-# Name or Organization <email address>
-# The email address is not required for organizations.
-
-Filippo Valsorda <hi@filippo.io>
-Google Inc.
-Ingo Oeser <nightlyone@googlemail.com>
-Palm Stone Games, Inc.
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -1,40 +0,0 @@
-# People who have agreed to one of the CLAs and can contribute patches.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# https://developers.google.com/open-source/cla/individual
-# https://developers.google.com/open-source/cla/corporate
-#
-# Names should be added to this file as:
-#     Name <email address>
-
-# Keep the list alphabetically sorted.
-
-Alexis Hunt <lexer@google.com>
-Andreas Litt <andreas.litt@gmail.com>
-Andrew Gerrand <adg@golang.org>
-Brad Fitzpatrick <bradfitz@golang.org>
-Burcu Dogan <jbd@google.com>
-Dave Day <djd@golang.org>
-David Sansome <me@davidsansome.com>
-David Symonds <dsymonds@golang.org>
-Filippo Valsorda <hi@filippo.io>
-Glenn Lewis <gmlewis@google.com>
-Ingo Oeser <nightlyone@googlemail.com>
-James Hall <james.hall@shopify.com>
-Johan Euphrosine <proppy@google.com>
-Jonathan Amsterdam <jba@google.com>
-Kunpei Sakai <namusyaka@gmail.com>
-Luna Duclos <luna.duclos@palmstonegames.com>
-Magnus Hiie <magnus.hiie@gmail.com>
-Mario Castro <mariocaster@gmail.com>
-Michael McGreevy <mcgreevy@golang.org>
-Omar Jarjur <ojarjur@google.com>
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Sarah Adams <shadams@google.com>
-Thanatat Tamtan <acoshift@gmail.com>
-Toby Burress <kurin@google.com>
-Tuo Shan <shantuo@google.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -137,7 +137,7 @@ func testOnGCE() bool {
 	resc := make(chan bool, 2)
 
 	// Try two strategies in parallel.
-	// See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194
+	// See https://github.com/googleapis/google-cloud-go/issues/194
 	go func() {
 		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
 		req.Header.Set("User-Agent", userAgent)
@@ -300,8 +300,8 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 		// being stable anyway.
 		host = metadataIP
 	}
-	url := "http://" + host + "/computeMetadata/v1/" + suffix
+	u := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
+	req, _ := http.NewRequest("GET", u, nil)
 	req.Header.Set("Metadata-Flavor", "Google")
 	req.Header.Set("User-Agent", userAgent)
 	res, err := c.hc.Do(req)
@@ -312,13 +312,13 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 	if res.StatusCode == http.StatusNotFound {
 		return "", "", NotDefinedError(suffix)
 	}
-	if res.StatusCode != 200 {
-		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
 	all, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", "", err
 	}
+	if res.StatusCode != 200 {
+		return "", "", &Error{Code: res.StatusCode, Message: string(all)}
+	}
 	return string(all), res.Header.Get("Etag"), nil
 }
 
@@ -499,3 +499,15 @@ func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) erro
 		}
 	}
 }
+
+// Error contains an error response from the server.
+type Error struct {
+	// Code is the HTTP response status code.
+	Code int
+	// Message is the server response message.
+	Message string
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message)
+}

backend/vendor/github.com/PuerkitoBio/goquery/.travis.yml

@@ -11,5 +11,6 @@ go:
     - 1.8.x
     - 1.9.x
     - "1.10.x"
+    - 1.11.x
     - tip
 

backend/vendor/github.com/PuerkitoBio/goquery/README.md

@@ -37,6 +37,8 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
+*    **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
 *    **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
 *    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
 *    **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
@@ -95,12 +97,24 @@ package main
 import (
   "fmt"
   "log"
+  "net/http"
 
   "github.com/PuerkitoBio/goquery"
 )
 
 func ExampleScrape() {
-  doc, err := goquery.NewDocument("http://metalsucks.net")
+  // Request the HTML page.
+  res, err := http.Get("http://metalsucks.net")
+  if err != nil {
+    log.Fatal(err)
+  }
+  defer res.Body.Close()
+  if res.StatusCode != 200 {
+    log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+  }
+
+  // Load the HTML document
+  doc, err := goquery.NewDocumentFromReader(res.Body)
   if err != nil {
     log.Fatal(err)
   }
@@ -126,6 +140,7 @@ func main() {
 - [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
 - [asciimoo/colly](https://github.com/asciimoo/colly), a lightning fast and elegant Scraping Framework
 - [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
+- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
 
 ## Support
 

backend/vendor/github.com/PuerkitoBio/goquery/go.mod

@@ -0,0 +1,6 @@
+module github.com/PuerkitoBio/goquery
+
+require (
+	github.com/andybalholm/cascadia v1.0.0
+	golang.org/x/net v0.0.0-20181114220301-adae6a3d119a
+)

backend/vendor/github.com/PuerkitoBio/goquery/go.sum

@@ -0,0 +1,5 @@
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

backend/vendor/github.com/coreos/bbolt/.gitignore

@@ -2,3 +2,4 @@
 *.test
 *.swp
 /bin/
+cover.out

backend/vendor/github.com/coreos/bbolt/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+go_import_path: go.etcd.io/bbolt
+
+sudo: false
+
+go:
+- 1.11
+
+before_install:
+- go get -v honnef.co/go/tools/...
+- go get -v github.com/kisielk/errcheck
+
+script:
+- make fmt
+- make test
+- make race
+# - make errcheck

backend/vendor/github.com/coreos/bbolt/Makefile

@@ -5,14 +5,34 @@ GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 default: build
 
 race:
-	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@TEST_FREELIST_TYPE=hashmap go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@echo "array freelist test"
+	@TEST_FREELIST_TYPE=array go test -v -race -test.run="TestSimulate_(100op|1000op)"
+
+fmt:
+	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
+
+# go get honnef.co/go/tools/simple
+gosimple:
+	gosimple ./...
+
+# go get honnef.co/go/tools/unused
+unused:
+	unused ./...
 
 # go get github.com/kisielk/errcheck
 errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/boltdb/bolt
+	@errcheck -ignorepkg=bytes -ignore=os:Remove go.etcd.io/bbolt
 
-test: 
+test:
-	@go test -v -cover .
+	TEST_FREELIST_TYPE=hashmap go test -timeout 20m -v -coverprofile cover.out -covermode atomic
-	@go test -v ./cmd/bolt
+	# Note: gets "program not an importable package" in out of path builds
+	TEST_FREELIST_TYPE=hashmap go test -v ./cmd/bbolt
 
-.PHONY: fmt test
+	@echo "array freelist test"
+
+	@TEST_FREELIST_TYPE=array go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	@TEST_FREELIST_TYPE=array go test -v ./cmd/bbolt
+
+.PHONY: race fmt errcheck test gosimple unused

backend/vendor/github.com/coreos/bbolt/README.md

@@ -1,5 +1,18 @@
-Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg)
+bbolt
-====
+=====
+
+[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt)
+[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt)
+[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt)
+[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt)
+[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases)
+[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE)
+
+bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value
+store. The purpose of this fork is to provide the Go community with an active
+maintenance and development target for Bolt; the goal is improved reliability
+and stability. bbolt includes bug fixes, performance enhancements, and features
+not found in Bolt while preserving backwards compatibility with the Bolt API.
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,
@@ -10,47 +23,55 @@ Since Bolt is meant to be used as such a low-level piece of functionality,
 simplicity is key. The API will be small and only focus on getting values
 and setting values. That's it.
 
+[gh_ben]: https://github.com/benbjohnson
+[bolt]: https://github.com/boltdb/bolt
 [hyc_symas]: https://twitter.com/hyc_symas
 [lmdb]: http://symas.com/mdb/
 
 ## Project Status
 
-Bolt is stable and the API is fixed. Full unit test coverage and randomized
+Bolt is stable, the API is fixed, and the file format is fixed. Full unit
-black box testing are used to ensure database consistency and thread safety.
+test coverage and randomized black box testing are used to ensure database
-Bolt is currently in high-load production environments serving databases as
+consistency and thread safety. Bolt is currently used in high-load production
-large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed
+environments serving databases as large as 1TB. Many companies such as
-services every day.
+Shopify and Heroku use Bolt-backed services every day.
+
+## Project versioning
+
+bbolt uses [semantic versioning](http://semver.org).
+API should not change between patch and minor releases.
+New minor versions may add additional features to the API.
 
 ## Table of Contents
 
-- [Getting Started](#getting-started)
+  - [Getting Started](#getting-started)
-  - [Installing](#installing)
+    - [Installing](#installing)
-  - [Opening a database](#opening-a-database)
+    - [Opening a database](#opening-a-database)
-  - [Transactions](#transactions)
+    - [Transactions](#transactions)
-    - [Read-write transactions](#read-write-transactions)
+      - [Read-write transactions](#read-write-transactions)
-    - [Read-only transactions](#read-only-transactions)
+      - [Read-only transactions](#read-only-transactions)
-    - [Batch read-write transactions](#batch-read-write-transactions)
+      - [Batch read-write transactions](#batch-read-write-transactions)
-    - [Managing transactions manually](#managing-transactions-manually)
+      - [Managing transactions manually](#managing-transactions-manually)
-  - [Using buckets](#using-buckets)
+    - [Using buckets](#using-buckets)
-  - [Using key/value pairs](#using-keyvalue-pairs)
+    - [Using key/value pairs](#using-keyvalue-pairs)
-  - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
+    - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
-  - [Iterating over keys](#iterating-over-keys)
+    - [Iterating over keys](#iterating-over-keys)
-    - [Prefix scans](#prefix-scans)
+      - [Prefix scans](#prefix-scans)
-    - [Range scans](#range-scans)
+      - [Range scans](#range-scans)
-    - [ForEach()](#foreach)
+      - [ForEach()](#foreach)
-  - [Nested buckets](#nested-buckets)
+    - [Nested buckets](#nested-buckets)
-  - [Database backups](#database-backups)
+    - [Database backups](#database-backups)
-  - [Statistics](#statistics)
+    - [Statistics](#statistics)
-  - [Read-Only Mode](#read-only-mode)
+    - [Read-Only Mode](#read-only-mode)
-  - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
+    - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
-- [Resources](#resources)
+  - [Resources](#resources)
-- [Comparison with other databases](#comparison-with-other-databases)
+  - [Comparison with other databases](#comparison-with-other-databases)
-  - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
+    - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
-  - [LevelDB, RocksDB](#leveldb-rocksdb)
+    - [LevelDB, RocksDB](#leveldb-rocksdb)
-  - [LMDB](#lmdb)
+    - [LMDB](#lmdb)
-- [Caveats & Limitations](#caveats--limitations)
+  - [Caveats & Limitations](#caveats--limitations)
-- [Reading the Source](#reading-the-source)
+  - [Reading the Source](#reading-the-source)
-- [Other Projects Using Bolt](#other-projects-using-bolt)
+  - [Other Projects Using Bolt](#other-projects-using-bolt)
 
 ## Getting Started
 
@@ -59,13 +80,28 @@ services every day.
 To start using Bolt, install Go and run `go get`:
 
 ```sh
-$ go get github.com/boltdb/bolt/...
+$ go get go.etcd.io/bbolt/...
 ```
 
 This will retrieve the library and install the `bolt` command line utility into
 your `$GOBIN` path.
 
 
+### Importing bbolt
+
+To use bbolt as an embedded key-value store, import as:
+
+```go
+import bolt "go.etcd.io/bbolt"
+
+db, err := bolt.Open(path, 0666, nil)
+if err != nil {
+  return err
+}
+defer db.Close()
+```
+
+
 ### Opening a database
 
 The top-level object in Bolt is a `DB`. It is represented as a single file on
@@ -79,7 +115,7 @@ package main
 import (
 	"log"
 
-	"github.com/boltdb/bolt"
+	bolt "go.etcd.io/bbolt"
 )
 
 func main() {
@@ -209,7 +245,7 @@ and then safely close your transaction if an error is returned. This is the
 recommended way to use Bolt transactions.
 
 However, sometimes you may want to manually start and end your transactions.
-You can use the `Tx.Begin()` function directly but **please** be sure to close
+You can use the `DB.Begin()` function directly but **please** be sure to close
 the transaction.
 
 ```go
@@ -395,7 +431,7 @@ db.View(func(tx *bolt.Tx) error {
 	c := tx.Bucket([]byte("MyBucket")).Cursor()
 
 	prefix := []byte("1234")
-	for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
 		fmt.Printf("key=%s, value=%s\n", k, v)
 	}
 
@@ -448,6 +484,10 @@ db.View(func(tx *bolt.Tx) error {
 })
 ```
 
+Please note that keys and values in `ForEach()` are only valid while
+the transaction is open. If you need to use a key or value outside of
+the transaction, you must use `copy()` to copy it to another byte
+slice.
 
 ### Nested buckets
 
@@ -460,6 +500,55 @@ func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error)
 func (*Bucket) DeleteBucket(key []byte) error
 ```
 
+Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings.
+
+```go
+
+// createUser creates a new user in the given account.
+func createUser(accountID int, u *User) error {
+    // Start the transaction.
+    tx, err := db.Begin(true)
+    if err != nil {
+        return err
+    }
+    defer tx.Rollback()
+
+    // Retrieve the root bucket for the account.
+    // Assume this has already been created when the account was set up.
+    root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10)))
+
+    // Setup the users bucket.
+    bkt, err := root.CreateBucketIfNotExists([]byte("USERS"))
+    if err != nil {
+        return err
+    }
+
+    // Generate an ID for the new user.
+    userID, err := bkt.NextSequence()
+    if err != nil {
+        return err
+    }
+    u.ID = userID
+
+    // Marshal and save the encoded user.
+    if buf, err := json.Marshal(u); err != nil {
+        return err
+    } else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil {
+        return err
+    }
+
+    // Commit the transaction.
+    if err := tx.Commit(); err != nil {
+        return err
+    }
+
+    return nil
+}
+
+```
+
+
+
 
 ### Database backups
 
@@ -469,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block
 your other database reads and writes.
 
 By default, it will use a regular file handle which will utilize the operating
-system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx)
+system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx)
 documentation for information about optimizing for larger-than-RAM datasets.
 
 One common use case is to backup over HTTP so you can use tools like `cURL` to
@@ -715,6 +804,9 @@ Here are a few things to note when evaluating and using Bolt:
   can be reused by a new page or can be unmapped from virtual memory and you'll
   see an `unexpected fault address` panic when accessing it.
 
+* Bolt uses an exclusive write lock on the database file so it cannot be
+  shared by multiple processes.
+
 * Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
   buckets that have random inserts will cause your database to have very poor
   page utilization.
@@ -755,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt:
 
 ## Reading the Source
 
-Bolt is a relatively small code base (<3KLOC) for an embedded, serializable,
+Bolt is a relatively small code base (<5KLOC) for an embedded, serializable,
 transactional key/value database so it can be a good starting point for people
 interested in how databases work.
 
@@ -807,46 +899,56 @@ them via pull request.
 
 Below is a list of public, open source projects that use Bolt:
 
-* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
+* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
-* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
 * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
-* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
+* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal.
-* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
+* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support.
-* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
+* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB
-* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
-* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
-* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
-* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
-* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
-* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
 * [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
-* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
+* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
-* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
+* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
-* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
-* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
 * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
-* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
+* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet.
-* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
-* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
-* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
-* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
-* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
-* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
-* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
-* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
-  backed by boltdb.
 * [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining
   simple tx and key scans.
-* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
+* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
-* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
+* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
+* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
+* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency.
+* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
+* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
+* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
 * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
-* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
-* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
-* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
 * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
-* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
-* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
-* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
 * [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter.
+* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains
+* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
+* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
+* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
+* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
+* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies
+* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
+* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding.
+* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
+* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
+* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
+* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
+* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
+* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
+* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
+* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
+* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi.
+* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
+* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
+  backed by boltdb.
+* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
+* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
+* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
+* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
+* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
+* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development.
+* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
 
 If you are using Bolt in a project please send a pull request to add it to the list.

backend/vendor/github.com/coreos/bbolt/appveyor.yml

@@ -1,18 +0,0 @@
-version: "{build}"
-
-os: Windows Server 2012 R2
-
-clone_folder: c:\gopath\src\github.com\boltdb\bolt
-
-environment:
-  GOPATH: c:\gopath
-
-install:
-  - echo %PATH%
-  - echo %GOPATH%
-  - go version
-  - go env
-  - go get -v -t ./...
-
-build_script:
-  - go test -v ./...

backend/vendor/github.com/coreos/bbolt/bolt_386.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_amd64.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_arm.go

@@ -1,7 +1,28 @@
-package bolt
+package bbolt
+
+import "unsafe"
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned bool
+
+func init() {
+	// Simple check to see whether this arch handles unaligned load/stores
+	// correctly.
+
+	// ARM9 and older devices require load/stores to be from/to aligned
+	// addresses. If not, the lower 2 bits are cleared and that address is
+	// read in a jumbled up order.
+
+	// See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+
+	raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11}
+	val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2))
+
+	brokenUnaligned = val != 0x11222211
+}

backend/vendor/github.com/coreos/bbolt/bolt_arm64.go

@@ -1,9 +1,12 @@
 // +build arm64
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false