make sure last returns [] list #262

* add widgets to demo page

* fix border color in dark mode

* demo page adjusting

* fix indentation

.dockerignore

@@ -5,7 +5,7 @@
 /web/public/
 /.vscode/
 /.idea/
-#/.git/objects/
+#/.git/
 
 # source files
 docker-compose.yml
@@ -19,3 +19,5 @@ debug
 debug.test
 *.prof
 *.test
+/bin/
+remark42

.drone.yml

@@ -7,14 +7,23 @@ pipeline:
   build:
     image: golang:1.10-alpine
     commands:
-      - cd app
+      - sleep 5
+      - nslookup mongo
+      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
+      - cat backend/.mongo
+      - cd backend/app
       - go build -v ./...
 
   docker_master:
     image: plugins/docker
     repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
+    secrets: [ docker_username, docker_password]
-    tags: 
+    build_args:
+      - DRONE=${DRONE}
+      - DRONE_TAG=${DRONE_TAG}
+      - DRONE_COMMIT=${DRONE_COMMIT}
+      - DRONE_BRANCH=${DRONE_BRANCH}
+    tags:
       - ${DRONE_COMMIT_BRANCH/\//-}
     when:
       branch: [master, release/*]
@@ -25,6 +34,7 @@ pipeline:
     repo: umputun/remark42
     secrets: [ docker_username, docker_password ]
     build_args:
+      - DRONE=${DRONE}
       - DRONE_TAG=${DRONE_TAG}
       - DRONE_COMMIT=${DRONE_COMMIT}
     tags: 
@@ -33,26 +43,32 @@ pipeline:
     when:
       event: tag
 
+  artifacts_tag:
+    image: plugins/docker
+    dockerfile: Dockerfile.artifacts
+    build_args:
+      - DRONE=${DRONE}
+      - DRONE_TAG=${DRONE_TAG}
+      - DRONE_COMMIT=${DRONE_COMMIT}
+      - GITHUB_TOKEN=${GITHUB_TOKEN}
+    when:
+      event: tag
+
   docker_branch:
     image: plugins/docker
     repo: umputun/remark42
     secrets: [ docker_username, docker_password ]
-    tags: 
+    build_args:
+      - DRONE=${DRONE}
+      - DRONE_COMMIT=${DRONE_COMMIT}
+      - DRONE_BRANCH=${DRONE_BRANCH}
+    tags:
       - ${DRONE_COMMIT_BRANCH/\//-}
     when:
       branch:
         exclude: [master, release/*]
         event: push
 
-  docker_pullrequest:
-    image: docker
-    commands:
-      - docker build .
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    when:
-      event: pull_request
-
   deploy:
     image: appleboy/drone-ssh
     host: remark42.com
@@ -78,3 +94,9 @@ pipeline:
     secrets: [ email_username, email_password ]
     when:
       status: [ changed, failure ]
+
+services:
+  mongo:
+    image: mongo:3.6
+    command: [ --smallfiles ]
+

.gitignore

@@ -10,7 +10,11 @@ debug.test
 .idea/
 /web/node_modules/
 /web/public/
+/web/coverage
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
+.mongo
+remark42
+/bin/

.travis.yml

@@ -3,6 +3,9 @@ install:
   - docker-compose --version
 
 script:
+  - docker run -d --name=mongo mongo:3.6 && sleep 3
+  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
+  - echo "running mongo on $MONGO_TEST"
   - docker build
       --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
       --build-arg CI=$CI
@@ -16,4 +19,7 @@ script:
       --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
       --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
       --build-arg TRAVIS_TAG=$TRAVIS_TAG
+      --build-arg MONGO_TEST=$MONGO_TEST
       .
+  - docker rm -f mongo
+  

Dockerfile

@@ -12,44 +12,76 @@ ARG TRAVIS_PULL_REQUEST
 ARG TRAVIS_PULL_REQUEST_SHA
 ARG TRAVIS_REPO_SLUG
 ARG TRAVIS_TAG
+ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+ARG DRONE_PULL_REQUEST
 
-WORKDIR /go/src/github.com/umputun/remark
+ARG SKIP_BACKEND_TEST
+ARG MONGO_TEST
 
-ADD app /go/src/github.com/umputun/remark/app
+WORKDIR /go/src/github.com/umputun/remark/backend
-ADD vendor /go/src/github.com/umputun/remark/vendor
+ADD backend /go/src/github.com/umputun/remark/backend
 ADD .git /go/src/github.com/umputun/remark/.git
 
-RUN cd app && go test ./...
+# run tests
+RUN \
+    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
+    cd app && \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
+    else echo "skip backend test" ; fi
 
-RUN gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
+RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-    --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./...
 
-RUN mkdir -p target && /script/coverage.sh
+# linters
+RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
+    gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
+    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
+    --enable=deadcode  --enable=gosimple --exclude=test --exclude=mock --exclude=vendor ./... ; \
+    else echo "skip backend linters" ; fi
 
+# coverage report
+RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
+    mkdir -p target && /script/coverage.sh ; \
+    else echo "skip backend coverage" ; fi
+
+# submit coverage to coverals if COVERALLS_TOKEN in env
 RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo coverall not enabled ; \
+    echo "coverall not enabled" ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi
+    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
 
+# if DRONE presented use DRONE_* git env to make version
 RUN \
-    version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) && \
+    if [ -z "$DRONE" ] ; then \
-    echo "git version=$version" && \  
+    echo "runs outside of drone" && version="local"; \
-    if [ -z "$DRONE_TAG" ] ; then \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
-    echo "runs outside of drone" ; \
+    echo "version=$version" && \
-    else version=${DRONE_TAG}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
-    echo "final version=$version" && \  
-    go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app
 
 
-FROM node:9.4-alpine as build-frontend
+FROM node:10.11-alpine as build-frontend-deps
+
+ARG CI
 
-ADD web /srv/web
 RUN apk add --no-cache --update git
-RUN \
+ADD web/package.json /srv/web/package.json
-    cd /srv/web && \
+ADD web/package-lock.json /srv/web/package-lock.json
-    npm i && npm run build && \
+RUN cd /srv/web && CI=true npm ci
+
+FROM node:10.11-alpine as build-frontend
+
+ARG CI
+ARG SKIP_FRONTEND_TEST
+ARG NODE_ENV=production
+
+COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+ADD web /srv/web
+RUN cd /srv/web && \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test build ; \
+    else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
 
@@ -57,21 +89,19 @@ FROM umputun/baseimage:app-latest
 
 WORKDIR /srv
 
-ADD scripts/import-disqus.sh /srv/import-disqus.sh
+ADD entrypoint.sh /entrypoint.sh
-ADD scripts/restore-backup.sh /srv/restore-backup.sh
+ADD backend/scripts/backup.sh /usr/local/bin/backup
-ADD scripts/migrate-data.sh /srv/migrate-data.sh
+ADD backend/scripts/restore.sh /usr/local/bin/restore
-ADD scripts/create-backup.sh /srv/create-backup.sh
+ADD backend/scripts/import.sh /usr/local/bin/import
+RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-ADD start.sh /srv/start.sh
+COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42 /srv/remark42
-
-RUN chmod +x /srv/start.sh /srv/import-disqus.sh /srv/restore-backup.sh /srv/migrate-data.sh /srv/create-backup.sh
-
-COPY --from=build-backend /go/src/github.com/umputun/remark/remark /srv/
 COPY --from=build-frontend /srv/web/public/ /srv/web
 RUN chown -R app:app /srv
+RUN ln -s /srv/remark42 /usr/bin/remark42
 
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["/srv/start.sh"]
+CMD ["server"]
-ENTRYPOINT ["/init.sh"]
+ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.artifacts

@@ -0,0 +1,96 @@
+FROM node:10.11-alpine as build-frontend-deps
+
+ARG CI
+ARG DRONE
+ARG DRONE_TAG
+ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+
+ENV SKIP_FRONTEND_TEST=true
+
+RUN apk add --no-cache --update git
+ADD web/package.json /srv/web/package.json
+ADD web/package-lock.json /srv/web/package-lock.json
+RUN cd /srv/web && CI=true npm ci
+
+FROM node:10.11-alpine as build-frontend
+
+ARG CI
+ARG NODE_ENV=production
+ENV SKIP_FRONTEND_TEST=true
+
+COPY --from=build-frontend-deps /srv/web/node_modules /srv/web/node_modules
+ADD web /srv/web
+RUN cd /srv/web && \
+    npm run build && \
+    rm -rf ./node_modules
+
+
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+ARG GITHUB_TOKEN
+ENV SKIP_BACKEND_TEST=true
+
+WORKDIR /go/src/github.com/umputun/remark/backend
+ADD backend /go/src/github.com/umputun/remark/backend
+ADD README.md /go/src/github.com/umputun/remark/
+ADD LICENSE /go/src/github.com/umputun/remark/
+COPY --from=build-frontend /srv/web/public/ web
+
+RUN \
+    export WEB_ROOT=/go/src/github.com/umputun/remark/backend/web && \
+    sed -i "s|https://demo.remark42.com|http://127.0.0.1:8080|g" ${WEB_ROOT}/*.js && \
+    sed -i "/REMOVE-START/,/REMOVE-END/d" ${WEB_ROOT}/iframe.html && \
+    go get -v github.com/rakyll/statik && \
+    statik --src=${WEB_ROOT} --dest=/go/src/github.com/umputun/remark/backend/app/rest -p api -f && \
+    ls -la /go/src/github.com/umputun/remark/backend/app/rest/api/statik.go && \
+    ls -la /go/src/github.com/umputun/remark/backend/web/
+
+# if DRONE presented use DRONE_* git env to make version
+RUN \
+    if [ -z "$DRONE" ] ; then \
+    echo "runs outside of drone" && version="local"; \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    echo "version=$version" && \
+    GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+
+RUN \
+    if [ -z "$DRONE_TAG" ] ; then \
+    echo "runs outside of drone" && tag=""; \
+    else tag=_${DRONE_TAG}; fi && \
+    apk add --no-cache --update zip && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 ../LICENSE ../README.md && \
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 ../LICENSE ../README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe ../LICENSE ../README.md
+
+# upload to github
+RUN \
+    if [ -z "$DRONE_TAG" ] ; then \
+    echo "skip upload to github" ; \
+    else \
+    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+
+FROM alpine
+COPY --from=build-backend /go/src/github.com/umputun/remark/backend/remark42.* /artifacts/
+RUN ls -la /artifacts/*
+CMD ["sleep", "100"]

Gopkg.lock

@@ -1,237 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter"
-  ]
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware"
-  ]
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/google/uuid"
-  packages = ["."]
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru"
-  ]
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-  packages = ["."]
-  revision = "0dc08b1671f34c4250ce212759ebd880f743d883"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64"
-  ]
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom"
-  ]
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex"
-  ]
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch"
-  ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "af8b7f1817ce6e82746722a745184bd50c341733bcc024311aa956ea32475796"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Makefile

@@ -0,0 +1,26 @@
+OS=linux
+ARCH=amd64
+
+bin:
+	docker build -f Dockerfile.artifacts -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
+	docker rm -f remark42.bin
+
+docker:
+	docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
+
+deploy:
+	docker build -f Dockerfile.artifacts -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	- @mkdir -p bin
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
+	docker rm -f remark42.bin
+
+.PHONY: bin

README.md

@@ -4,7 +4,7 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 
 * Social login via Google, Facebook, Github and Yandex
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus
+* Import from disqus and wordpress
 * Markdown support
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
@@ -14,53 +14,129 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
+* Self-contained executable can be deployed directly to Linux, Windows and MacOS
 * Clean, lightweight and fully customizable UI
 * Multi-site mode from a single instance
-* Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le)
+* Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+
+#
+
+  - [Install](#install)
+    - [Backend](#backend)
+      - [With Docker](#with-docker) 
+      - [Without docker](#without-docker) 
+      - [Parameters](#parameters)
+        - [Required parameters](#required-parameters)
+      - [Register oauth2 providers](#register-oauth2-providers)
+        - [Google Auth Provider](#google-auth-provider)
+        - [GitHub Auth Provider](#github-auth-provider)
+        - [Facebook Auth Provider](#facebook-auth-provider)
+        - [Yandex Auth Provider](#yandex-auth-provider)
+      - [Initial import from Disqus](#initial-import-from-disqus)
+      - [Initial import from WordPress](#initial-import-from-wordpress)
+      - [Backup and restore](#backup-and-restore)
+        - [Automatic backups](#automatic-backups)
+        - [Manual backup](#manual-backup)
+        - [Restore from backup](#restore-from-backup)
+        - [Backup format](#backup-format)
+      - [Admin users](#admin-users)
+    - [Setup on your website](#setup-on-your-website)
+      - [Comments](#comments)
+      - [Last comments](#last-comments)
+      - [Counter](#counter)
+  - [Build from the source](#build-from-the-source)
+  - [Development](#development)
+    - [Backend development](#backend-development)
+    - [Frontend development](#frontend-development)
+      - [Build](#build)
+      - [Devserver](#devserver)
+  - [API](#api)
+    - [Authorization](#authorization)
+    - [Commenting](#commenting)
+    - [RSS feeds](#rss-feeds)
+    - [Admin](#admin)
+  - [Privacy](#privacy)
+  - [Technical details](#technical-details)
+
+
 ## Install
 
 ### Backend
 
+#### With Docker
+
+_this is the recommended way to run remark42_
+
 * copy provided `docker-compose.yml` and customize for your needs
-* prepare user id for container `` export USER=`id -u $USER` ``
 * make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
-* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
+* pull prepared images from the docker hub and start - `docker-compose pull && docker-compose up -d`
-* alternatively compile from sources - `docker-compose build && docker-compose up -d`
+* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
+
+#### Without docker
+
+* download archive for [stable release](https://github.com/umputun/remark/releases) or [development version](https://remark42.com/downloads)
+* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows) 
+* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
+* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm32]`
 
 #### Parameters
 
-| Command line       | Environment        | Default               | Description                                    |
+| Command line            | Environment             | Default               | Description                                      |
-| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
+| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                | REMARK_URL         |                       | url to remark42 server, _required_             |
+| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret             | SECRET             |                       | secret key, _required_                         |
+| secret                  | SECRET                  |                       | secret key, _required_                           |
-| store.bolt.path    | STORE_BOLT_PATH    | `./var`               | path to data directory                         |
+| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s`                 | boltdb access timeout                          |
+| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| site               | SITE               | `remark`              | site name(s), _multi_                          |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| admin              | ADMIN              |                       | admin names (list of user ids), _multi_        |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| admin-email        | ADMIN_EMAIL        | `admin@${REMARK_URL}` | admin email                                    |
+| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| backup             | BACKUP_PATH        | `./var/backup`        | backups location                               |
+| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| max-back           | MAX_BACKUP_FILES   | `10`                  | max backup files to keep                       |
+| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| cache.max.items    | CACHE_MAX_ITEMS    | `1000`                | max number of cached items, `0` - unlimited    |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| cache.max.value    | CACHE_MAX_VALUE    | `65536`               | max size of cached value, `0` - unlimited      |
+| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| cache.max.size     | CACHE_MAX_SIZE     | `50000000`            | max size of all cached values, `0` - unlimited |
+| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| avatar.path        | AVATAR_FS_PATH     | `./var/avatars`       | avatars location                               |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| avatar.rsz-lmt     | AVATAR_RSZ_LMT     | 0                     | max image size for resizing avatars on save    |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| max-comment        | MAX_COMMENT_SIZE   | 2048                  | comment's size limit                           |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| auth.google.cid    | AUTH_GOOGLE_CID    |                       | Google OAuth client ID                         |
+| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| auth.google.csec   | AUTH_GOOGLE_CSEC   |                       | Google OAuth client secret                     |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| auth.facebook.cid  | AUTH_FACEBOOK_CID  |                       | Facebook OAuth client ID                       |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| auth.facebook.csec | AUTH_FACEBOOK_CSEC |                       | Facebook OAuth client secret                   |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | 0                     | max image size for resizing avatars on save      |
-| auth.github.cid    | AUTH_GITHUB_CID    |                       | Github OAuth client ID                         |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | 5m                    | jwt TTL                                          |
-| auth.github.csec   | AUTH_GITHUB_CSEC   |                       | Github OAuth client secret                     |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | 200h                  | cookie TTL                                       |
-| auth.yandex.cid    | AUTH_YANDEX_CID    |                       | Yandex OAuth client ID                         |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.yandex.csec   | AUTH_YANDEX_CSEC   |                       | Yandex OAuth client secret                     |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| low-score          | LOW_SCORE          | `-5`                  | Low score threshold                            |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| critical-score     | CRITICAL_SCORE     | `-10`                 | Critical score threshold                       |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| img-proxy          | IMG_PROXY          | `false`               | Enable http->https proxy for images            |
+| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| dbg                | DEBUG              | `false`               | debug mode                                     |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| dev-passwd         | DEV_PASSWD         |                       | password for `dev` user                        |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
+| auth.dev                | AUTH_DEV                | false                 | local oauth2 server, development mode only       |
+| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
+| notify.queue            | NOTIFY_QUEUE            | 100                   | size of notification queue                       |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT |                       | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | 8443                  | port for https server                            |
+| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | 2048                  | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
+| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
+| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
+| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            |                       | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`               | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -71,8 +147,8 @@ Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engi
 Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
-1. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
+2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
-2. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
+3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
 
 The minimal `docker-compose.yml` has to include all required parameters:
 
@@ -81,15 +157,15 @@ version: '2'
 
 services:
     remark42:
-        image: umputun/remark42:master
+        image: umputun/remark42:latest
         restart: always
         container_name: "remark42"
         environment:
             - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
             - SECRET=abcd-123456-xyz-$%^&           # secret key
             - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
             - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
-            - USER=1001                             # UID on the host machine, i.e `id -u`
         volumes:
             - ./var:/srv/var                        # persistent volume to store all remark42 data 
 ```
@@ -150,7 +226,13 @@ For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/co
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
 2.  Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Run import command - `docker-compose exec remark42 /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
+3.  Run import command - `docker exec -it remark42 import -p disqus -f {disqus-export-name}.xml -s {your site id}`
+
+#### Initial import from WordPress
+
+1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments. 
+2. Move this file to your remark42 host within `./var`
+3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
 
 #### Backup and restore
 
@@ -159,20 +241,19 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./
 
 For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
 
-`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}`
+`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
-
-##### Schema migration
-
-One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema 
-and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance.
-
-`docker-compose exec remark42 /srv/migrate-data.sh {your site id}`
 
 ##### Manual backup
 
-In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz`
+In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
 
-`docker-compose exec remark42 /srv/create-backup.sh {your site id}`
+`docker exec -it remark42 backup -s {your site id}`
+
+##### Restore from backup
+
+Restore will clean all comments first and then will processed with complete import from a given file.
+
+`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
 
 ##### Backup format
 
@@ -185,37 +266,15 @@ Admins/moderators should be defined in `docker-compose.yml` as a list of user ID
 
 ```
     environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
+        - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
 To get user id just login and click on your username or any other user you want to promote to admins. 
 It will expand login info and show full user ID.
 
+### Setup on your website
 
-### Frontend
+#### Comments
-
-Frontend part is building automatically along with backend if you use `docker-compose`.
-
-For manual building:
-
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* run `npm install` inside `./web`;
-* run `npm run build` there;
-* result files will be saved in `./web/public`.
-
-For development mode use `npm start` instead of `npm run build`.
-In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
-
-URLs for development:
-
-* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
-* `localhost:8080/dev.html` — page with embedded script from local folder;
-* `localhost:8080/last-comments.html` — page with embedded script for last comments;
-* `localhost:8080/counter.html` — page with embedded script for counter with examples.
-
-#### Usage
-
-##### Comments
 
 It's a main widget which renders list of comments. 
 
@@ -226,7 +285,9 @@ Add this snippet to the bottom of web page:
   var remark_config = {
     site_id: 'YOUR_SITE_ID',
     url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
-    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used 
+    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
+    theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
+    page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
   (function() {
@@ -245,7 +306,20 @@ And then add this node in the place where you want to see Remark42 widget:
 
 After that widget will be rendered inside this node.
 
-##### Last comments
+##### Themes
+
+Right now Remark has two themes: light and dark.
+You can pick one using configuration object,
+but there is also a possibility to switch between themes in runtime.
+For this purpose Remark adds to `window` object named `REMARK42`, 
+which contains function `changeTheme`.
+Just call this function and pass a name of the theme that you want to turn on:
+
+```js
+window.REMARK42.changeTheme('light');
+```      
+
+#### Last comments
 
 It's a widget which renders list of last comments from your site.
 
@@ -273,7 +347,7 @@ And then add this node in the place where you want to see last comments widget:
 
 `data-max` sets the max amount of comments (default: `15`).
 
-##### Counter
+#### Counter
 
 It's a widget which renders a number of comments for the specified page.
 
@@ -282,7 +356,7 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
+    site_id: 'YOUR_SITE_ID',
   };
 
   (function() {
@@ -305,11 +379,80 @@ and it will use `data-url` attribute to define the page with comments.
 
 Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
 
+## Build from the source
+
+- to build docker container - `make docker`. This command will produce container `umputun/remark42`.
+- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable 
+ `remark42` file with everything embedded.
+ 
+## Development
+
+You can use fully functional local version to develop and test both frontend & backend.
+
+To bring it up run:
+
+```bash
+# if you mainly work on backend
+docker-compose -f compose-dev-backend.yml build
+docker-compose -f compose-dev-backend.yml up
+# if you mainly work on frontend
+docker-compose -f compose-dev-frontend.yml build
+docker-compose -f compose-dev-frontend.yml up
+```
+
+It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
+To access UI demo page go to `127.0.0.1:8080/web`.
+By default, you would be logged in as `dev_user` which defined as admin. 
+You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
+
+Backend docker compose config by default skips running frontend related tests. 
+Frontend docker compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build. 
+
+### Backend development
+
+In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
+
+To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
+It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
+`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
+
+To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
+`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
+
+### Frontend development
+
+#### Build
+
+* install [Node.js 8](https://nodejs.org/en/) or higher;
+* install [NPM 6.1.0](https://www.npmjs.com/package/npm);
+* run `npm install` inside `./web`;
+* run `npm run build` there;
+* result files will be saved in `./web/public`.
+
+**Note** Running `npm install` will set up precommit hooks into your git repository.
+It used to reformat your frontend code using `prettier` and lint with `eslint` before every commit.
+
+#### Devserver
+
+For local development mode with Hot Reloading use `npm start` instead of `npm run build`.
+In this case `webpack` will serve files using `webpack-dev-server` on `localhost:9000`.
+By visiting `127.0.0.1:9000/web` you will get a page with main comments widget
+communicating with demo server backend running on `https://demo.remark42.com`.
+But you will not be able to login with any oauth providers due to security reasons.
+
+You can attach to locally running backend by providing `REMARK_URL` environment variable.
+```sh
+npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
+```
+
+Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
+[Redux](https://github.com/zalmoxisus/redux-devtools-extension).
+
 ## API
 
 ### Authorization
 
-* `GET /auth/{provider}/login?from=http://url&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
+* `GET /auth/{provider}/login?from=http://url&site=site_id&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
 * `GET /auth/logout` - logout
 
 ```go
@@ -342,6 +485,7 @@ type Comment struct {
     Timestamp time.Time       `json:"time"`    // time stamp, read only
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
+    PostTitle string          `json:"title"`   // post title
 }
 
 type Locator struct {
@@ -371,15 +515,14 @@ type Node struct {
 
 Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
 
-* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in 5min since creation
+* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
-```json
+```go
-  Content-Type: application/json
+ 	type EditRequest struct {
-
+ 		Text    string `json:"text"`    // updated text
-  {
+ 		Summary string `json:"summary"` // optional, summary of the edit
-    "text": "edit comment blah http://radio-t.com 12345",
+ 		Delete  bool   `json:"delete"`  // delete flag 
-    "summary": "fix blah"
+ 	}{}
-  }
 ```
 
 * `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
@@ -392,6 +535,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
   }{}
   ```
 * `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
+* `POST /api/v1/count?site=siteID` - get number of comments for posts from post body (list of post IDs)
 * `GET /api/v1/list?site=site-id&limit=5&skip=2` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
   ```go
   type PostInfo struct {
@@ -424,58 +568,61 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
   
 * `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
+* `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
-* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1` - block or unblock user.
+* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1&ttl=7d` - block or unblock user with optional ttl (default=permanent)
-* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids.
+* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids
   ```go
   type BlockedUser struct {
       ID        string    `json:"id"`
       Name      string    `json:"name"`
-      Timestamp time.Time `json:"time"`
+      Until     time.Time `json:"time"`
   }
   ```
 * `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup.
+* `POST /api/v1/admin/import?site=side-id` - import comments from the backup, uses post body.
+* `POST /api/v1/admin/import/form?site=side-id` - import comments from the backup, user post form.
+* `GET /api/v1/admin/import/wait?site=side-id` - wait for import completeion.
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
-* `DELETE /api/v1/admin/user/{userid}?site=site-id&block=1` - delete all user's comments.
+* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
+* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
 * `PUT /api/v1/admin/readonly?site=site-id&url=post-url&ro=1` - set read-only status
 * `PUT /api/v1/admin/verify/{userid}?site=site-id&verified=1` - set verified status
 * `GET /api/v1/admin/deleteme?token=token` - process deleteme user's request
 
 _all admin calls require auth and admin privilege_
 
-
 ## Privacy 
 
 * Remark42 is trying to be very sensitive to any private or semi-private information.
-* Authentication requesting the lowest (minimal) possible scope from providers. All extra information returned by them dropped immediately and not stored in any form.
+* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
-* Generally remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
+* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
 * There is no tracking of any sort.
-* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing Cross-Site Request Forgery
+* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF.
 * There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
 * There are no third-party analytic services involved.
 * User can request all information remark42 knows about and export to gz file.
-* Supported complete cleanup of all information related to user activity on demand.
+* Supported complete cleanup of all information related to user's activity.
 * Cookie lifespan can be restricted to session-only. 
 * All potentially sensitive data stored by remark42 hashed and encrypted.
 
-
 ## Technical details
 
-* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `BOLTDB_PATH`
+* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `STORE_BOLT_PATH`
 * Each site stored in a separate boltbd file.
-* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_STORE` should be transferred.
+* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_FS_PATH` should be transferred. Optionally, boltdb can be used to store avatars as well.
 * Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
-* Authentication implemented with [jwt](https://github.com/dgrijalva/jwt-go) stored in a cookie. It uses HttpOnly, secure cookies.
+* Authentication implemented with [go-pkgz/auth](https://github.com/go-pkgz/auth) stored in a cookie. It uses HttpOnly, secure cookies.
-* All heavy REST calls cached internally in LRU cache limited by `MAX_CACHE_ITEMS` and `MAX_CACHE_SIZE`.
+* All heavy REST calls cached internally in LRU cache limited by `CACHE_MAX_ITEMS` and `CACHE_MAX_SIZE` with [go-pkgz/rest](https://github.com/go-pkgz/rest)
 * User's activity throttled globally (up to 1000 simultaneous requests) and limited locally (per user, usually up to 10 req/sec)
 * Request timeout set to 60sec
-* Development mode (`--dev-password` set) allows to test remark42 without social login and with admin privileges. Adds basic-auth for username: `dev`, password: `${DEV_PASSWD}`. **should not be used in production deployment**
+* Admin authentication (`--admin-password` set) allows to hit remark42 API without social login and with admin privileges. Adds basic-auth for username: `admin`, password: `${ADMIN_PASSWD}`.
-* User can vote for the comment multiple times but only to change his/her vote. Double-voting not allowed.
+* User can vote for the comment multiple times but only to change the vote. Double-voting not allowed.
-* User can edit comments in 5 mins window after creation.
+* User can edit comments in 5 mins (configurable) window after creation.
 * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
-* All avatars cached locally to prevent rate limiters from google/github/facebook/yandex.
+* All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
+* Images can be proxied (`IMG_PROXY=true`) to prevent mixed http/https.
 * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.

app/main.go

@@ -1,398 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/hashicorp/logutils"
-	"github.com/jessevdk/go-flags"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/api"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Opts with command line flags and env
-// nolint:maligned
-type Opts struct {
-	SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
-	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
-
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-
-	Sites          []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	Admins         []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
-	AdminEmail     string   `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
-	DevPasswd      string   `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
-	BackupLocation string   `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles int      `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy     bool     `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize int      `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	LowScore       int      `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore  int      `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	ReadOnlyAge    int      `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
-	Port           int      `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot        string   `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	Dbg            bool     `long:"dbg" env:"DEBUG" description:"debug mode"`
-
-	Auth struct {
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
-}
-
-// AuthGroup defines options group for auth params
-type AuthGroup struct {
-	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
-	CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
-}
-
-// StoreGroup defines options group for store params
-type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
-	Bolt struct {
-		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
-		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
-	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
-}
-
-// AvatarGroup defines options group for avatar params
-type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
-	FS   struct {
-		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
-	} `group:"fs" namespace:"fs" env-namespace:"FS"`
-	RszLmt int `long:"rsz-lmt" env:"RSZ_LMT" default:"0" description:"max image size for resizing avatars on save"`
-}
-
-// CacheGroup defines options group for cache params
-type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
-	Max  struct {
-		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
-		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
-		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
-	} `group:"max" namespace:"max" env-namespace:"MAX"`
-}
-
-var revision = "unknown"
-
-// Application holds all active objects
-type Application struct {
-	Opts
-	restSrv     *api.Rest
-	migratorSrv *api.Migrator
-	exporter    migrator.Exporter
-	terminated  chan struct{}
-}
-
-func main() {
-	fmt.Printf("remark %s\n", revision)
-
-	var opts Opts
-	p := flags.NewParser(&opts, flags.Default)
-	if _, e := p.ParseArgs(os.Args[1:]); e != nil {
-		os.Exit(1)
-	}
-
-	setupLog(opts.Dbg)
-	log.Print("[INFO] started remark")
-	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() { // catch signal and invoke graceful termination
-		stop := make(chan os.Signal, 1)
-		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
-		<-stop
-		log.Print("[WARN] interrupt signal")
-		cancel()
-	}()
-
-	app, err := New(opts)
-	if err != nil {
-		log.Fatalf("[ERROR] failed to setup application, %+v", err)
-	}
-	err = app.Run(ctx)
-	log.Printf("[INFO] remark terminated %s", err)
-}
-
-// New prepares application and return it with all active parts
-// doesn't start anything
-func New(opts Opts) (*Application, error) {
-
-	if err := makeDirs(opts.BackupLocation); err != nil {
-		return nil, err
-	}
-
-	if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
-		return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
-	}
-
-	boltStore, err := makeDataStore(opts.Store, opts.Sites)
-	if err != nil {
-		return nil, err
-	}
-	dataService := &service.DataStore{
-		Interface:      boltStore,
-		EditDuration:   5 * time.Minute,
-		Secret:         opts.SecretKey,
-		MaxCommentSize: opts.MaxCommentSize,
-	}
-
-	loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
-		cache.MaxKeys(opts.Cache.Max.Items), cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
-	if err != nil {
-		return nil, err
-	}
-
-	jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour)
-
-	avatarStore, err := makeAvatarStore(opts.Avatar)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make avatar store")
-	}
-	avatarProxy := &proxy.Avatar{
-		Store:     avatarStore,
-		RoutePath: "/api/v1/avatar",
-		RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
-	}
-
-	exporter := &migrator.Remark{DataStore: dataService}
-
-	migr := &api.Migrator{
-		Version:        revision,
-		Cache:          loadingCache,
-		NativeImporter: &migrator.Remark{DataStore: dataService},
-		DisqusImporter: &migrator.Disqus{DataStore: dataService},
-		NativeExported: &migrator.Remark{DataStore: dataService},
-		SecretKey:      opts.SecretKey,
-	}
-
-	srv := &api.Rest{
-		Version:     revision,
-		DataService: dataService,
-		Exporter:    exporter,
-		WebRoot:     opts.WebRoot,
-		RemarkURL:   opts.RemarkURL,
-		ImageProxy:  &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
-		AvatarProxy: avatarProxy,
-		ReadOnlyAge: opts.ReadOnlyAge,
-		Authenticator: auth.Authenticator{
-			JWTService: jwtService,
-			Admins:     opts.Admins,
-			AdminEmail: opts.AdminEmail,
-			Providers:  makeAuthProviders(jwtService, avatarProxy, dataService, opts),
-			DevPasswd:  opts.DevPasswd,
-		},
-		Cache: loadingCache,
-	}
-
-	// no admin email, use admin@domain
-	if srv.Authenticator.AdminEmail == "" {
-		if u, err := url.Parse(opts.RemarkURL); err == nil {
-			srv.Authenticator.AdminEmail = "admin@" + u.Host
-		}
-	}
-
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
-	tch := make(chan struct{})
-	return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, Opts: opts, terminated: tch}, nil
-}
-
-// Run all application objects
-func (a *Application) Run(ctx context.Context) error {
-	if a.DevPasswd != "" {
-		log.Printf("[WARN] running in dev mode")
-	}
-
-	go func() {
-		// shutdown on context cancellation
-		<-ctx.Done()
-		a.restSrv.Shutdown()
-		a.migratorSrv.Shutdown()
-	}()
-	a.activateBackup(ctx) // runs in goroutine for each site
-	go a.migratorSrv.Run(a.Port + 1)
-	a.restSrv.Run(a.Port)
-	close(a.terminated)
-	return nil
-}
-
-// Wait for application completion (termination)
-func (a *Application) Wait() {
-	<-a.terminated
-}
-
-// activateBackup runs background backups for each site
-func (a *Application) activateBackup(ctx context.Context) {
-	for _, siteID := range a.Sites {
-		backup := migrator.AutoBackup{
-			Exporter:       a.exporter,
-			BackupLocation: a.BackupLocation,
-			SiteID:         siteID,
-			KeepMax:        a.MaxBackupFiles,
-			Duration:       24 * time.Hour,
-		}
-		go backup.Do(ctx)
-	}
-}
-
-// makeDataStore creates store for all sites
-func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
-	switch group.Type {
-	case "bolt":
-		if err = makeDirs(group.Bolt.Path); err != nil {
-			return nil, err
-		}
-		sites := []engine.BoltSite{}
-		for _, site := range siteNames {
-			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
-		}
-		result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
-	default:
-		return nil, errors.Errorf("unsupported store type %s", group.Type)
-	}
-
-	return result, errors.Wrap(err, "can't initialize data store")
-}
-
-func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
-	switch group.Type {
-	case "fs":
-		if err = makeDirs(group.FS.Path); err != nil {
-			return nil, err
-		}
-		return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
-	}
-	return nil, errors.Errorf("unsupported avatart store type %s", group.Type)
-}
-
-// mkdir -p for all dirs
-func makeDirs(dirs ...string) error {
-
-	// exists returns whether the given file or directory exists or not
-	exists := func(path string) (bool, error) {
-		_, err := os.Stat(path)
-		if err == nil {
-			return true, nil
-		}
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return true, err
-	}
-
-	for _, dir := range dirs {
-		ex, err := exists(dir)
-		if err != nil {
-			return errors.Wrapf(err, "can't check directory status for %s", dir)
-		}
-		if !ex {
-			if e := os.MkdirAll(dir, 0700); e != nil {
-				return errors.Wrapf(err, "can't make directory %s", dir)
-			}
-		}
-	}
-	return nil
-}
-
-func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
-
-	makeParams := func(cid, secret string) auth.Params {
-		return auth.Params{
-			JwtService:   jwtService,
-			AvatarProxy:  avatarProxy,
-			RemarkURL:    opts.RemarkURL,
-			Cid:          cid,
-			Csecret:      secret,
-			Admins:       opts.Admins,
-			SecretKey:    opts.SecretKey,
-			IsVerifiedFn: ds.IsVerifiedFn(),
-		}
-	}
-
-	providers := []auth.Provider{}
-	if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
-		providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
-	}
-	if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
-		providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
-	}
-	if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
-		providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
-	}
-	if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
-		providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
-	}
-	if len(providers) == 0 {
-		log.Printf("[WARN] no auth providers defined")
-	}
-	return providers
-}
-
-// post-flush callback invoked by cache after each flush in async way
-func postFlushFn(sites []string, port int) func() {
-
-	return func() {
-		// list of heavy urls for pre-heating on cache change
-		urls := []string{
-			"http://localhost:%d/api/v1/list?site=%s",
-			"http://localhost:%d/api/v1/last/50?site=%s",
-		}
-
-		for _, site := range sites {
-			for _, u := range urls {
-				resp, err := http.Get(fmt.Sprintf(u, port, site))
-				if err != nil {
-					log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
-					return
-				}
-				if err = resp.Body.Close(); err != nil {
-					log.Printf("[WARN] failed to close response body, %s", err)
-				}
-			}
-		}
-	}
-}
-
-func resetEnv(envs ...string) {
-	for _, env := range envs {
-		if err := os.Unsetenv(env); err != nil {
-			log.Printf("[WARN] can't unset env %s, %s", env, err)
-		}
-	}
-}
-
-func setupLog(dbg bool) {
-	filter := &logutils.LevelFilter{
-		Levels:   []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},
-		MinLevel: logutils.LogLevel("INFO"),
-		Writer:   os.Stdout,
-	}
-
-	log.SetFlags(log.Ldate | log.Ltime)
-
-	if dbg {
-		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
-		filter.MinLevel = logutils.LogLevel("DEBUG")
-	}
-	log.SetOutput(filter)
-}

app/main_test.go

@@ -1,135 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	flags "github.com/jessevdk/go-flags"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestApplication(t *testing.T) {
-	app, ctx := prepApp(t, 18080, 500*time.Millisecond)
-	go func() { _ = app.Run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
-
-	// add comment
-	resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
-		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
-	require.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	body, _ = ioutil.ReadAll(resp.Body)
-	t.Log(string(body))
-
-	assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
-
-	app.Wait()
-}
-
-func TestApplicationFailed(t *testing.T) {
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-
-	// RO bolt location
-	_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/dev/null"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
-		"open /dev/null/remark.db: not a directory")
-	t.Log(err)
-
-	// RO backup location
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
-		"--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
-	t.Log(err)
-
-	// invalid url
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
-	t.Log(err)
-
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.type=mongo"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "unsupported store type mongo")
-	t.Log(err)
-}
-
-func TestApplicationShutdown(t *testing.T) {
-	app, ctx := prepApp(t, 18090, 500*time.Millisecond)
-	st := time.Now()
-	err := app.Run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-	app.Wait()
-}
-
-func TestApplicationMainSignal(t *testing.T) {
-	os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
-		"--port=18100", "--url=https://demo.remark42.com"}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
-	}()
-	st := time.Now()
-	main()
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-}
-
-func prepApp(t *testing.T, port int, duration time.Duration) (*Application, context.Context) {
-	// prepare options
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
-	require.Nil(t, err)
-	opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
-	opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", port)
-	opts.Store.Bolt.Timeout = 10 * time.Second
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
-	opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
-	opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
-	opts.Port = port
-
-	os.Remove(opts.Store.Bolt.Path + "/remark.db")
-
-	// create app
-	app, err := New(opts)
-	require.Nil(t, err)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	return app, ctx
-}

app/migrator/remark.go

@@ -1,95 +0,0 @@
-package migrator
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"io"
-	"log"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/store"
-)
-
-// Remark implements exporter and importer for internal store format
-type Remark struct {
-	DataStore Store
-}
-
-// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
-func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
-	topics, err := r.DataStore.List(siteID, 0, 0)
-	if err != nil {
-		return 0, err
-	}
-	log.Printf("[DEBUG] exporting %d topics", len(topics))
-
-	commentsCount := 0
-	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
-		topic := topics[i]
-		comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
-			return commentsCount, err
-		}
-
-		for _, comment := range comments {
-
-			buf := &bytes.Buffer{}
-			enc := json.NewEncoder(buf)
-			enc.SetEscapeHTML(false)
-
-			if err := enc.Encode(comment); err != nil {
-				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
-			}
-			data := buf.Bytes()
-			if _, err := w.Write(data); err != nil {
-				return commentsCount, errors.Wrap(err, "can't write comment data")
-			}
-			commentsCount++
-		}
-	}
-	log.Printf("[DEBUG] exported %d comments", commentsCount)
-	return commentsCount, nil
-}
-
-// Import comments from json strings produced by Remark.Export
-func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
-
-	if err := r.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
-	}
-
-	failed := 0
-	total, comments := 0, 0
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		rec := scanner.Bytes()
-		if len(rec) < 3 {
-			continue
-		}
-		total++
-		comment := store.Comment{}
-		if err := json.Unmarshal(rec, &comment); err != nil {
-			failed++
-			log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
-			continue
-		}
-		if _, err := r.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
-	}
-	if scanner.Err() != nil {
-		return comments, errors.Wrap(scanner.Err(), "error in scan")
-	}
-	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
-	}
-	log.Printf("[INFO] imported %d comments from %d records", comments, total)
-	return comments, nil
-}

app/migrator/remark_test.go

@@ -1,112 +0,0 @@
-package migrator
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestRemark_Export(t *testing.T) {
-	b := prep(t)
-	r := Remark{DataStore: b}
-
-	buf := &bytes.Buffer{}
-	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	c1, err := buf.ReadString('\n')
-	assert.Nil(t, err)
-	log.Print(c1)
-	exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-	assert.Equal(t, exp, c1)
-}
-
-func TestRemark_Import(t *testing.T) {
-	r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	buf.WriteString(r1)
-	buf.WriteString(r2)
-	buf.WriteString("{}")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	size, err := r.Import(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[0].ID)
-	assert.Equal(t, "afbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ParentID)
-}
-
-func TestRemark_ImportManyWithError(t *testing.T) {
-	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < 1200; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
-	}
-	buf.WriteString("bad1\n")
-	buf.WriteString("bad2\n")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	n, err := r.Import(buf, "radio-t")
-	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-
-	b := &service.DataStore{Interface: boltStore}
-
-	comment := store.Comment{
-		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/rest/api/middleware.go

@@ -1,169 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"regexp"
-	"runtime/debug"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi/middleware"
-
-	"github.com/umputun/remark/app/rest"
-)
-
-// JSON is a map alias, just for convenience
-type JSON map[string]interface{}
-
-// AppInfo adds custom app-info to the response header
-func AppInfo(app string, version string) func(http.Handler) http.Handler {
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Org", "Umputun")
-			w.Header().Set("App-Name", app)
-			w.Header().Set("App-Version", version)
-			if mhost := os.Getenv("MHOST"); mhost != "" {
-				w.Header().Set("Host", mhost)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// Ping middleware response with pong to /ping. Stops chain if ping request detected
-func Ping(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/ping") {
-			w.Header().Set("Content-Type", "text/plain")
-			w.WriteHeader(http.StatusOK)
-			if _, err := w.Write([]byte("pong")); err != nil {
-				log.Printf("[WARN] can't send pong, %s", err)
-			}
-			return
-		}
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// Recoverer is a middleware that recovers from panics, logs the panic and returns a HTTP 500 status if possible.
-func Recoverer(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		defer func() {
-			if rvr := recover(); rvr != nil {
-				log.Printf("[WARN] request panic, %v", rvr)
-				debug.PrintStack()
-				http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			}
-		}()
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// LoggerFlag type
-type LoggerFlag int
-
-// logger flags enum
-const (
-	LogAll LoggerFlag = iota
-	LogUser
-	LogBody
-	LogNone
-)
-const maxBody = 1024
-
-var reMultWhtsp = regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-// Logger middleware prints http log. Customized by set of LoggerFlag
-func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if inLogFlags(LogNone, flags) { // skip logging
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			ww := middleware.NewWrapResponseWriter(w, 1)
-			body, user := getBodyAndUser(r, flags)
-			t1 := time.Now()
-			defer func() {
-				t2 := time.Now()
-
-				q := r.URL.String()
-				if qun, err := url.QueryUnescape(q); err == nil {
-					q = qun
-				}
-
-				remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-				if strings.HasPrefix(r.RemoteAddr, "[") {
-					remoteIP = strings.Split(r.RemoteAddr, "]:")[0] + "]"
-				}
-				if ipFn != nil {
-					remoteIP = ipFn(remoteIP)
-				}
-
-				log.Printf("[INFO] REST %s - %s - %s - %d (%d) - %v %s %s",
-					r.Method, q, remoteIP, ww.Status(), ww.BytesWritten(), t2.Sub(t1), user, body)
-			}()
-
-			h.ServeHTTP(ww, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-
-	return f
-}
-
-func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user string) {
-	ctx := r.Context()
-	if ctx == nil {
-		return "", ""
-	}
-
-	if inLogFlags(LogBody, flags) {
-		if content, err := ioutil.ReadAll(r.Body); err == nil {
-			body = string(content)
-			r.Body = ioutil.NopCloser(bytes.NewReader(content))
-
-			if len(body) > 0 {
-				body = strings.Replace(body, "\n", " ", -1)
-				body = reMultWhtsp.ReplaceAllString(body, " ")
-			}
-
-			if len(body) > maxBody {
-				body = body[:maxBody] + "..."
-			}
-		}
-	}
-
-	if inLogFlags(LogUser, flags) {
-		u, err := rest.GetUserInfo(r)
-		if err == nil && u.Name != "" {
-			user = fmt.Sprintf(" - %s %q", u.ID, u.Name)
-		}
-	}
-
-	return body, user
-}
-
-func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool {
-	for _, flg := range flags {
-		if (flg == LogAll && f != LogNone) || flg == f {
-			return true
-		}
-	}
-	return false
-}

app/rest/api/middleware_test.go

@@ -1,63 +0,0 @@
-package api
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestMiddleware_AppInfo(t *testing.T) {
-	router := chi.NewRouter()
-	router.With(AppInfo("remark42", "12345")).Get("/blah", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
-		w.Write([]byte("blah blah"))
-	})
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/blah")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "blah blah", string(b))
-	assert.Equal(t, "remark42", resp.Header.Get("App-Name"))
-	assert.Equal(t, "12345", resp.Header.Get("App-Version"))
-	assert.Equal(t, "Umputun", resp.Header.Get("Org"))
-}
-
-func TestMiddleware_GetBodyAndUser(t *testing.T) {
-	req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body"))
-	require.Nil(t, err)
-
-	body, user := getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, "body", body)
-	assert.Equal(t, "", user, "no user")
-
-	req = rest.SetUserInfo(req, store.User{ID: "id1", Name: "user1"})
-	_, user = getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-
-	body, user = getBodyAndUser(req, nil)
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogNone})
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogUser})
-	assert.Equal(t, "", body)
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-}

app/rest/api/migrator.go

@@ -1,133 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-)
-
-// Migrator rest runs on unexposed port and available for local requests only
-type Migrator struct {
-	Version        string
-	Cache          cache.LoadingCache
-	NativeImporter migrator.Importer
-	DisqusImporter migrator.Importer
-	NativeExported migrator.Exporter
-	SecretKey      string
-
-	httpServer *http.Server
-	lock       sync.Mutex
-}
-
-// Run the listener and request's router, activate rest server
-// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
-func (m *Migrator) Run(port int) {
-	log.Printf("[INFO] activate import server on port %d", port)
-	router := m.routes()
-
-	m.lock.Lock()
-	m.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
-	m.lock.Unlock()
-
-	err := m.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown import http server
-func (m *Migrator) Shutdown() {
-	log.Print("[WARN] shutdown import server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-
-	m.lock.Lock()
-	if err := m.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] importer shutdown error, %s", err)
-	}
-	m.lock.Unlock()
-
-	log.Print("[DEBUG] shutdown import server completed")
-}
-
-func (m *Migrator) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(15*time.Minute))
-	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-	router.Use(AppInfo("remark42-migrator", m.Version), Ping, Logger(nil, LogAll))
-	router.Post("/api/v1/admin/import", m.importCtrl)
-	router.Get("/api/v1/admin/export", m.exportCtrl)
-	return router
-}
-
-// POST /import?secret=key&site=site-id&provider=disqus|remark
-// imports comments from post body.
-func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-	importer := m.NativeImporter
-	if r.URL.Query().Get("provider") == "disqus" {
-		importer = m.DisqusImporter
-	}
-
-	size, err := importer.Import(r.Body, siteID)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
-		return
-	}
-	m.Cache.Flush(siteID)
-
-	render.Status(r, http.StatusCreated)
-	render.JSON(w, r, JSON{"status": "ok", "size": size})
-}
-
-// GET /export?site=site-id&secret=12345
-// exports all comments for siteID as gz file
-func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-
-	exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-	w.WriteHeader(http.StatusOK)
-	gzWriter := gzip.NewWriter(w)
-	defer func() {
-		if e := gzWriter.Close(); e != nil {
-			log.Printf("[WARN] can't close gzip writer, %s", e)
-		}
-	}()
-
-	if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
-}

app/rest/api/migrator_test.go

@@ -1,126 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-func TestMigrator_Import(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b))
-}
-
-func TestMigrator_ImportRejected(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-}
-
-func TestMigrator_Export(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	require.Nil(t, err)
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=123456", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
-	t.Logf("%s", string(ungzBody))
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=bad", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 403, resp.StatusCode)
-}
-
-func TestMigrator_Shutdown(t *testing.T) {
-	srv := Migrator{}
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b}
-	svc = &Migrator{
-		DisqusImporter: &migrator.Disqus{DataStore: dataStore},
-		NativeImporter: &migrator.Remark{DataStore: dataStore},
-		NativeExported: &migrator.Remark{DataStore: dataStore},
-		Cache:          &mockCache{},
-		SecretKey:      "123456",
-	}
-
-	routes := svc.routes()
-	ts = httptest.NewServer(routes)
-	return svc, ts
-}
-
-func cleanupImportSrv(srv *Migrator, ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-}

app/rest/api/rest.go

@@ -1,249 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-	"github.com/pkg/errors"
-	"gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Rest is a rest access server
-type Rest struct {
-	Version         string
-	DataService     *service.DataStore
-	Authenticator   auth.Authenticator
-	Exporter        migrator.Exporter
-	Cache           cache.LoadingCache
-	AvatarProxy     *proxy.Avatar
-	ImageProxy      *proxy.Image
-	WebRoot         string
-	RemarkURL       string
-	ReadOnlyAge     int
-	ScoreThresholds struct {
-		Low      int
-		Critical int
-	}
-
-	httpServer *http.Server
-	lock       sync.Mutex
-
-	adminService admin
-}
-
-const hardBodyLimit = 1024 * 64 // limit size of body
-
-var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
-	blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
-	blackfriday.BackslashLineBreak | blackfriday.Autolink
-
-type commentsWithInfo struct {
-	Comments []store.Comment `json:"comments"`
-	Info     store.PostInfo  `json:"info,omitempty"`
-}
-
-// Run the lister and request's router, activate rest server
-func (s *Rest) Run(port int) {
-	log.Printf("[INFO] activate rest server on port %d", port)
-
-	if len(s.Authenticator.Admins) > 0 {
-		log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
-	}
-
-	router := s.routes()
-
-	s.lock.Lock()
-	s.httpServer = &http.Server{
-		Addr:              fmt.Sprintf(":%d", port),
-		Handler:           router,
-		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      5 * time.Second,
-		IdleTimeout:       30 * time.Second,
-	}
-	s.lock.Unlock()
-
-	err := s.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown rest http server
-func (s *Rest) Shutdown() {
-	log.Print("[WARN] shutdown rest server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	s.lock.Lock()
-	if err := s.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] rest shutdown error, %s", err)
-	}
-	log.Print("[DEBUG] shutdown rest server completed")
-	s.lock.Unlock()
-}
-
-func (s *Rest) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
-	router.Use(AppInfo("remark42", s.Version), Ping)
-
-	s.adminService = admin{
-		dataService:   s.DataService,
-		exporter:      s.Exporter,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
-
-	ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization
-
-	// auth routes for all providers
-	router.Route("/auth", func(r chi.Router) {
-		r.Use(Logger(ipFn, LogAll), tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
-		for _, provider := range s.Authenticator.Providers {
-			r.Mount("/"+provider.Name, provider.Routes()) // mount auth providers as /auth/{name}
-		}
-		if len(s.Authenticator.Providers) > 0 {
-			// shortcut, can be any of providers, all logouts do the same - removes cookie
-			r.Get("/logout", s.Authenticator.Providers[0].LogoutHandler)
-		}
-	})
-
-	avatarMiddlewares := []func(http.Handler) http.Handler{
-		Logger(ipFn, LogNone),
-		tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)),
-	}
-	router.Mount(s.AvatarProxy.Routes(avatarMiddlewares...)) // mount avatars to /api/v1/avatar/{file.img}
-
-	// api routes
-	router.Route("/api/v1", func(rapi chi.Router) {
-		rapi.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-
-		// open routes
-		rapi.Group(func(ropen chi.Router) {
-			ropen.Use(s.Authenticator.Auth(false))
-			ropen.Use(Logger(ipFn, LogAll))
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
-			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
-			ropen.Get("/info", s.infoCtrl)
-
-			ropen.Mount("/rss", s.rssRoutes())
-			ropen.Mount("/img", s.ImageProxy.Routes())
-		})
-
-		// protected routes, require auth
-		rapi.Group(func(rauth chi.Router) {
-			rauth.Use(s.Authenticator.Auth(true))
-			rauth.Use(Logger(ipFn, LogAll))
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
-
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll)))
-		})
-	})
-
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
-	return router
-}
-
-// serves static files from /web
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
-	log.Printf("[INFO] run file server for %s, path %s", root, path)
-	origPath := path
-	fs := http.StripPrefix(path, http.FileServer(root))
-	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
-		path += "/"
-	}
-	path += "*"
-
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// don't show dirs, just serve files
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-				http.NotFound(w, r)
-				return
-			}
-			fs.ServeHTTP(w, r)
-		}))
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) {
-	data, err := encodeJSONWithHTML(v)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't render json response")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-func encodeJSONWithHTML(v interface{}) ([]byte, error) {
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false)
-	if err := enc.Encode(v); err != nil {
-		return nil, errors.Wrap(err, "json encoding failed")
-	}
-	return buf.Bytes(), nil
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONFromBytes(w http.ResponseWriter, r *http.Request, data []byte) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	if status, ok := r.Context().Value(render.StatusCtxKey).(int); ok {
-		w.WriteHeader(status)
-	}
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
-	for _, c := range comments {
-		if fn(c) {
-			filtered = append(filtered, c)
-		}
-	}
-	return filtered
-}

app/rest/api/rest_test.go

@@ -1,146 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-var testHTML = "/tmp/test-remark.html"
-
-func TestRest_FileServer(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/web/test-remark.html")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, "some html", body)
-}
-
-func TestRest_Shutdown(t *testing.T) {
-	srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300),
-		RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, Secret: "123456"}
-	srv = &Rest{
-		DataService: dataStore,
-		Authenticator: auth.Authenticator{
-			DevPasswd:  "password",
-			Providers:  nil,
-			Admins:     []string{"a1", "a2"},
-			AdminEmail: "admin@remark-42.com",
-			JWTService: auth.NewJWT("12345", false, time.Minute),
-		},
-		Exporter:    &migrator.Remark{DataStore: dataStore},
-		Cache:       &mockCache{},
-		WebRoot:     "/tmp",
-		RemarkURL:   "https://demo.remark42.com",
-		AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300), RoutePath: "/api/v1/avatar"},
-		ImageProxy:  &proxy.Image{},
-		ReadOnlyAge: 10,
-	}
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
-
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-	ts = httptest.NewServer(srv.routes())
-	return srv, ts
-}
-
-func get(t *testing.T, url string) (string, int) {
-	r, err := http.Get(url)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func getWithAuth(t *testing.T, url string) (string, int) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	r, err := client.Do(req)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func post(t *testing.T, url string, body string) (*http.Response, error) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	return client.Do(req)
-}
-
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
-
-	b, err := json.Marshal(c)
-	assert.Nil(t, err, "can't marshal comment %+v", c)
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	crResp := JSON{}
-	err = json.Unmarshal(b, &crResp)
-	assert.Nil(t, err)
-	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
-}
-
-func cleanup(ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-	os.Remove(testHTML)
-}
-
-type mockCache struct{}
-
-func (mc *mockCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
-	return fn()
-}
-
-func (mc *mockCache) Flush(scopes ...string) {}

app/rest/api/rss_test.go

@@ -1,181 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestServer_RssPost(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-        <channel>
-            <title>Remark42 comments</title>
-            <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
-            <pubDate>%s</pubDate>
-            <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-            </item>
-         </channel>
-	</rss>`, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssSite(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah11#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/site?site=bad-radio-t")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssWithReply(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	c2.ParentID = id1
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one &gt; developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-}
-
-func waitOnSecChange() {
-	for {
-		if time.Now().Nanosecond() < 100000000 {
-			break
-		}
-		time.Sleep(10 * time.Nanosecond)
-	}
-}
-
-// clean formatting, i.e. multiple spaces, \t, \n
-func cleanRssFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/auth/auth.go

@@ -1,132 +0,0 @@
-// Package auth provides oauth2 support as well as related middlewares.
-package auth
-
-import (
-	"encoding/base64"
-	"log"
-	"net/http"
-	"strings"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Authenticator is top level auth object providing middlewares
-type Authenticator struct {
-	JWTService *JWT
-	Providers  []Provider
-	Admins     []string
-	AdminEmail string
-	DevPasswd  string
-}
-
-var devUser = store.User{
-	ID:      "dev",
-	Name:    "developer one",
-	Picture: "/api/v1/avatar/remark.image",
-	Admin:   true,
-}
-
-// Auth middleware adds auth from session and populates user info
-func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if a.basicDevUser(w, r) { // fail-back to dev user if enabled
-				user := devUser
-				r = rest.SetUserInfo(r, user)
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			claims, err := a.JWTService.Get(r)
-			if err != nil && reqAuth { // in full auth lack of session causes Unauthorized
-				log.Printf("[DEBUG] failed auth, %s", err)
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if err != nil { // in anonymous mode just pass it to the next handler
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			if claims.User == nil && reqAuth {
-				log.Print("[DEBUG] failed auth, no user info presented in the claim")
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if claims.User != nil { // if uinfo in token populate it to context
-				user := *claims.User
-				user.Admin = isAdmin(user.ID, a.Admins) // dbl-check for admin to reset admin flag even if token has it
-				// refresh token if it close to expiration
-				if _, err := a.JWTService.Refresh(w, r); err != nil {
-					log.Printf("[DEBUG] can't refresh jwt, %s", err)
-				}
-				r = rest.SetUserInfo(r, user)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// AdminOnly allows access to admins
-func (a *Authenticator) AdminOnly(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		user, err := rest.GetUserInfo(r)
-		if err != nil {
-			http.Error(w, "Unauthorized", http.StatusUnauthorized)
-			return
-		}
-
-		if !user.Admin {
-			http.Error(w, "Access denied", http.StatusForbidden)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) bool {
-
-	if a.DevPasswd == "" {
-		return false
-	}
-
-	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
-	if len(s) != 2 {
-		return false
-	}
-
-	b, err := base64.StdEncoding.DecodeString(s[1])
-	if err != nil {
-		return false
-	}
-
-	pair := strings.SplitN(string(b), ":", 2)
-	if len(pair) != 2 {
-		return false
-	}
-
-	if pair[0] != "dev" || pair[1] != a.DevPasswd {
-		return false
-	}
-
-	return true
-}
-
-func isAdmin(userID string, admins []string) bool {
-	for _, admin := range admins {
-		if admin == userID {
-			return true
-		}
-	}
-	return false
-}

app/rest/auth/auth_test.go

@@ -1,171 +0,0 @@
-package auth
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAuthJWTCookie(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	expiration := int(time.Duration(365 * 24 * time.Hour).Seconds())
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "wrong id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "xsrf mismatch")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtExpired, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "token expired")
-}
-
-func TestAuthJWTHeader(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtValid)
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtExpired)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "invalid auth token")
-}
-func TestAuthRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "xyz")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAuthNotRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(false)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "ZZZZ123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAdminRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user, admin")
-
-	devUser.Admin = false
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 403, resp.StatusCode, "valid auth user, not admin")
-
-}
-func withBasicAuth(r *http.Request, username, password string) *http.Request {
-	auth := username + ":" + password
-	r.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
-	return r
-}

app/rest/auth/jwt.go

@@ -1,166 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"time"
-
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// JWT wraps jwt operations
-// supports both header and cookie jwt
-type JWT struct {
-	secret        string
-	secureCookies bool
-	exp           time.Duration
-}
-
-// CustomClaims stores user info for auth and state & from from login
-type CustomClaims struct {
-	jwt.StandardClaims
-	User *store.User `json:"user,omitempty"`
-
-	// state and from used for oauth handshake
-	State       string `json:"state,omitempty"`
-	From        string `json:"from,omitempty"`
-	SiteID      string `json:"site_id,omitempty"`
-	SessionOnly bool   `json:"sess_only,omitempty"`
-}
-
-const jwtCookieName = "JWT"
-const jwtHeaderKey = "X-JWT"
-const xsrfCookieName = "XSRF-TOKEN"
-const xsrfHeaderKey = "X-XSRF-TOKEN"
-
-// NewJWT makes JWT service
-func NewJWT(secret string, secureCookies bool, exp time.Duration) *JWT {
-	res := JWT{
-		secret:        secret,
-		secureCookies: secureCookies,
-		exp:           exp,
-	}
-	return &res
-}
-
-// Token makes jwt with claims
-func (j *JWT) Token(claims *CustomClaims) (string, error) {
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
-	tokenString, err := token.SignedString([]byte(j.secret))
-	if err != nil {
-		return "", errors.Wrap(err, "can't sign jwt token")
-	}
-	return tokenString, nil
-}
-
-// Parse token string and verify
-func (j *JWT) Parse(tokenString string) (*CustomClaims, error) {
-	token, err := jwt.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) {
-		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
-			return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"])
-		}
-		return []byte(j.secret), nil
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "can't parse jwt")
-	}
-
-	claims, ok := token.Claims.(*CustomClaims)
-	if !ok || !token.Valid {
-		return nil, errors.New("invalid jwt")
-	}
-	return claims, nil
-}
-
-// Set creates jwt cookie with xsrf cookie and put it to ResponseWriter
-// accepts claims and sets expiration if none defined. permanent flag means long-living cookie, false makes it session only.
-func (j *JWT) Set(w http.ResponseWriter, claims *CustomClaims, sessionOnly bool) error {
-	if claims.ExpiresAt == 0 {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-	}
-
-	tokenString, err := j.Token(claims)
-	if err != nil {
-		return errors.Wrap(err, "failed to make jwt token")
-	}
-
-	cookieExpiration := 0 // session cookie
-	if !sessionOnly {
-		cookieExpiration = 365 * 24 * 3600 // 1 year
-	}
-
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: tokenString, HttpOnly: true, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: claims.Id, HttpOnly: false, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-
-	return nil
-}
-
-// Get jwt from header or cookie
-// if cookie used, verify xsrf token to match
-func (j *JWT) Get(r *http.Request) (*CustomClaims, error) {
-
-	fromCookie := false
-	tokenString := ""
-
-	// try to get from X-JWT header
-	if tokenHeader := r.Header.Get(jwtHeaderKey); tokenHeader != "" {
-		tokenString = tokenHeader
-	}
-
-	// try to get from JWT cookie
-	if tokenString == "" {
-		fromCookie = true
-		jc, err := r.Cookie(jwtCookieName)
-		if err != nil {
-			return nil, errors.Wrap(err, "jwt cookie was not presented")
-		}
-		tokenString = jc.Value
-	}
-
-	claims, err := j.Parse(tokenString)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get jwt")
-	}
-
-	if fromCookie && claims.User != nil {
-		xsrf := r.Header.Get(xsrfHeaderKey)
-		if claims.Id != xsrf {
-			return nil, errors.New("xsrf mismatch")
-		}
-	}
-
-	return claims, nil
-}
-
-// Refresh gets jwt from request, checks if it will be expiring soon (1/2 of expiration) and create the new onw
-func (j *JWT) Refresh(w http.ResponseWriter, r *http.Request) (*CustomClaims, error) {
-	claims, err := j.Get(r)
-	if err != nil {
-		return nil, err
-	}
-	untilExp := claims.ExpiresAt - time.Now().Unix()
-	if untilExp <= int64(j.exp.Seconds()/2) {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-		e := j.Set(w, claims, claims.SessionOnly)
-		return claims, e
-	}
-	return claims, nil
-}
-
-// Reset token's cookies
-func (j *JWT) Reset(w http.ResponseWriter) {
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-}

app/rest/auth/jwt_test.go

@@ -1,287 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCI" +
-	"sImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZS" +
-	"I6IiIsImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ._loFgh3g45gr9TtGqvM3N584I_6EHEOJnYb6Py84stQ"
-
-var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6In" +
-	"JlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsIm" +
-	"FkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWV9.p6w0sM_NYaRuyhyA9jqfWlB5cx1vZPGhXGC5geSX7nA"
-
-var testJwtExpired = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MjY4ODc4MjIsImp0aSI6InJhbmRvbSBpZCIs" +
-	"ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiI" +
-	"sImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ.4_dCrY9ihyfZIedz-kZwBTxmxU1a52V7IqeJrOqTzE4"
-
-func TestJWT_Token(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	res, err := j.Token(claims)
-	assert.Nil(t, err)
-	assert.Equal(t, testJwtValid, res)
-}
-
-func TestJWT_Parse(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-	claims, err := j.Parse(testJwtValid)
-	assert.NoError(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1"}, claims.User)
-
-	_, err = j.Parse(testJwtExpired)
-	assert.NotNil(t, err, "expired token")
-
-	_, err = j.Parse("bad")
-	assert.NotNil(t, err, "bad token")
-}
-
-func TestJWT_Set(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	claims.SessionOnly = false
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValid, cookies[0].Value)
-	assert.Equal(t, 31536000, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-
-	claims.SessionOnly = true
-	rr = httptest.NewRecorder()
-	err = j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies = rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValidSess, cookies[0].Value)
-	assert.Equal(t, 0, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-}
-
-func TestJWT_GetFromHeader(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	req := httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtValid)
-	claims, err := j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtExpired)
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, "bad bad token")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token contains an invalid number of segments"), err.Error())
-
-}
-
-func TestJWT_SetAndGetWithCookies(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State:       "123456",
-		From:        "from",
-		SessionOnly: true,
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	claims, err = j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-	assert.Equal(t, true, claims.SessionOnly)
-	t.Log(resp.Cookies())
-}
-
-func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id wrong")
-	claims, err = j.Get(req)
-	assert.EqualError(t, err, "xsrf mismatch")
-}
-
-func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2018, 5, 21, 1, 35, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/expired" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/expired")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/expired", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-}
-
-func TestJWT_Refresh(t *testing.T) {
-	j := NewJWT("xyz 12345", false, 2*time.Second)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:     "random id",
-			Issuer: "remark42",
-		},
-	}
-	// set token
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, true)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	require.Equal(t, 2, len(cookies))
-
-	req, err := http.NewRequest("GET", "http://example.com/blah", nil)
-	require.Nil(t, err)
-	req.AddCookie(cookies[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-
-	claims2, err := j.Refresh(rr, req)
-	require.Nil(t, err)
-	assert.Equal(t, claims.ExpiresAt, claims2.ExpiresAt, "no refresh yet")
-
-	time.Sleep(1 * time.Second)
-	claims2, err = j.Refresh(rr, req)
-	assert.Nil(t, err)
-	assert.True(t, claims.ExpiresAt < claims2.ExpiresAt, "refreshed")
-	t.Log(claims.ExpiresAt, claims2.ExpiresAt)
-}

app/rest/auth/provider.go

@@ -1,224 +0,0 @@
-package auth
-
-import (
-	"context"
-	"crypto/rand"
-	"crypto/sha1"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-)
-
-// Provider represents oauth2 provider
-type Provider struct {
-	Params
-	Name        string
-	RedirectURL string
-	InfoURL     string
-	Endpoint    oauth2.Endpoint
-	Scopes      []string
-	MapUser     func(userData, []byte) store.User // map info from InfoURL to User
-	conf        oauth2.Config
-}
-
-// Params to make initialized and ready to use provider
-type Params struct {
-	RemarkURL    string
-	AvatarProxy  *proxy.Avatar
-	JwtService   *JWT
-	IsVerifiedFn func(siteID string, userID string) bool
-	SecretKey    string
-	Admins       []string
-	Cid          string
-	Csecret      string
-}
-
-type userData map[string]interface{}
-
-func (u userData) value(key string) string {
-	// json.Unmarshal converts json "null" value to go's "nil", in this case return empty string
-	if val, ok := u[key]; ok && val != nil {
-		return fmt.Sprintf("%v", val)
-	}
-	return ""
-}
-
-// newProvider makes auth for given provider
-func initProvider(p Params, provider Provider) Provider {
-	log.Printf("[INFO] init auth provider %s", provider.Name)
-	provider.Params = p
-	provider.conf = oauth2.Config{
-		ClientID:     provider.Cid,
-		ClientSecret: provider.Csecret,
-		RedirectURL:  provider.RedirectURL,
-		Scopes:       provider.Scopes,
-		Endpoint:     provider.Endpoint,
-	}
-
-	log.Printf("[DEBUG] created %s auth, id=%s, redir=%s, endpoint=%s",
-		provider.Name, provider.Cid, provider.Endpoint, provider.RedirectURL)
-	return provider
-}
-
-// Routes returns auth routes for given provider
-func (p Provider) Routes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/login", p.loginHandler)
-	router.Get("/callback", p.authHandler)
-	router.Get("/logout", p.LogoutHandler)
-	return router
-}
-
-// loginHandler - GET /login?from=redirect-back-url&site=siteID&session=1
-func (p Provider) loginHandler(w http.ResponseWriter, r *http.Request) {
-
-	log.Printf("[DEBUG] login with %s", p.Name)
-	// make state (random) and store in session
-	state := p.randToken()
-
-	claims := CustomClaims{
-		State:       state,
-		From:        r.URL.Query().Get("from"),
-		SiteID:      r.URL.Query().Get("site"),
-		SessionOnly: r.URL.Query().Get("session") != "" && r.URL.Query().Get("session") != "0",
-		StandardClaims: jwt.StandardClaims{
-			Id:        p.randToken(),
-			Issuer:    "remark42",
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-		},
-	}
-
-	if err := p.JwtService.Set(w, &claims, false); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set jwt")
-		return
-	}
-
-	// return login url
-	loginURL := p.conf.AuthCodeURL(state)
-	log.Printf("[DEBUG] login url %s", loginURL)
-
-	http.Redirect(w, r, loginURL, http.StatusFound)
-}
-
-// authHandler fills user info and redirects to "from" url. This is callback url redirected locally by browser
-// GET /callback
-func (p Provider) authHandler(w http.ResponseWriter, r *http.Request) {
-
-	oauthClaims, err := p.JwtService.Get(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to get jwt")
-		return
-	}
-
-	retrievedState := oauthClaims.State
-	if retrievedState == "" || retrievedState != r.URL.Query().Get("state") {
-		http.Error(w, fmt.Sprintf("unexpected state %v", retrievedState), http.StatusUnauthorized)
-		return
-	}
-
-	log.Printf("[DEBUG] auth with state %s", retrievedState)
-	tok, err := p.conf.Exchange(context.Background(), r.URL.Query().Get("code"))
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "exchange failed")
-		return
-	}
-
-	client := p.conf.Client(context.Background(), tok)
-	uinfo, err := client.Get(p.InfoURL)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, fmt.Sprintf("failed to get client info via %s", p.InfoURL))
-		return
-	}
-
-	defer func() {
-		if e := uinfo.Body.Close(); e != nil {
-			log.Printf("[WARN] failed to close response body, %s", e)
-		}
-	}()
-
-	data, err := ioutil.ReadAll(uinfo.Body)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to read user info")
-		return
-	}
-
-	jData := map[string]interface{}{}
-	if e := json.Unmarshal(data, &jData); e != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to unmarshal user info")
-		return
-	}
-	log.Printf("[DEBUG] got raw user info %+v", jData)
-
-	u := p.MapUser(jData, data)
-	u = p.alterUser(u, oauthClaims)
-
-	authClaims := &CustomClaims{
-		User: &u,
-		StandardClaims: jwt.StandardClaims{
-			Issuer: "remark42",
-			Id:     p.randToken(),
-		},
-		SessionOnly: oauthClaims.SessionOnly,
-	}
-
-	if err = p.JwtService.Set(w, authClaims, oauthClaims.SessionOnly); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to save user info")
-		return
-	}
-
-	log.Printf("[DEBUG] user info %+v", u)
-
-	// redirect to back url if presented in login query params
-	if oauthClaims.From != "" {
-		http.Redirect(w, r, oauthClaims.From, http.StatusTemporaryRedirect)
-		return
-	}
-	render.JSON(w, r, &u)
-}
-
-// alterUser sets fields not handled by provider's MapUser, things like avatar, admin, verified
-func (p Provider) alterUser(u store.User, oauthClaims *CustomClaims) store.User {
-	if p.AvatarProxy != nil {
-		if avatarURL, e := p.AvatarProxy.Put(u); e == nil {
-			u.Picture = avatarURL
-		} else {
-			log.Printf("[WARN] failed to proxy avatar, %s", e)
-		}
-	}
-	u.Admin = isAdmin(u.ID, p.Admins)
-	if p.IsVerifiedFn != nil {
-		u.Verified = p.IsVerifiedFn(oauthClaims.SiteID, u.ID)
-	}
-	return u
-}
-
-// LogoutHandler - GET /logout
-func (p Provider) LogoutHandler(w http.ResponseWriter, r *http.Request) {
-	p.JwtService.Reset(w)
-	log.Printf("[DEBUG] logout")
-}
-
-func (p Provider) randToken() string {
-	b := make([]byte, 32)
-	if _, err := rand.Read(b); err != nil {
-		log.Fatalf("[ERROR] can't get randoms, %s", err)
-	}
-	s := sha1.New()
-	if _, err := s.Write(b); err != nil {
-		log.Printf("[WARN] can't write randoms, %s", err)
-	}
-	return fmt.Sprintf("%x", s.Sum(nil))
-}

app/rest/auth/provider_test.go

@@ -1,218 +0,0 @@
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/http/cookiejar"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestLogin(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, permanent
-	resp, err := client.Get("http://localhost:8981/login?site=remark")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	t.Logf("resp %s", string(body))
-	t.Logf("headers: %+v", resp.Header)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 31536000, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	u := store.User{}
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser1", Picture: "http://exmple.com/pic1.png",
-		Admin: false, Blocked: false, IP: ""}, u)
-
-	// check admin user
-	resp, err = client.Get("http://localhost:8981/login?site=remark")
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser2", Picture: "http://exmple.com/pic1.png",
-		Admin: true, Blocked: false, IP: "", Verified: true}, u)
-}
-
-func TestLoginSessionOnly(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, session
-	resp, err := client.Get("http://localhost:8981/login?site=remark&session=1")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 0, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	req, err := http.NewRequest("GET", "http://example.com", nil)
-	require.Nil(t, err)
-	req.AddCookie(resp.Cookies()[0])
-	req.AddCookie(resp.Cookies()[1])
-	req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value)
-
-	jwtService := NewJWT("12345", false, time.Hour)
-	res, err := jwtService.Get(req)
-	require.Nil(t, err)
-	assert.Equal(t, true, res.SessionOnly)
-	t.Logf("%+v", res)
-}
-
-func TestLogout(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8691, 8692)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	resp, err := client.Get("http://localhost:8691/login")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	resp, err = client.Get("http://localhost:8691/logout")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name, "jwt cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[0].Value)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name, "xsrf cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[1].Value)
-}
-
-func TestInitProvider(t *testing.T) {
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret"}
-	provider := Provider{Name: "test", RedirectURL: "redir"}
-	res := initProvider(params, provider)
-	assert.Equal(t, "cid", res.conf.ClientID)
-	assert.Equal(t, "csecret", res.conf.ClientSecret)
-	assert.Equal(t, "redir", res.RedirectURL)
-	assert.Equal(t, "123456", res.SecretKey)
-	assert.Equal(t, "test", res.Name)
-}
-
-func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Server) {
-
-	provider := Provider{
-		Name: "mock",
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  fmt.Sprintf("http://localhost:%d/login/oauth/authorize", authPort),
-			TokenURL: fmt.Sprintf("http://localhost:%d/login/oauth/access_token", authPort),
-		},
-		RedirectURL: fmt.Sprintf("http://localhost:%d/callback", loginPort),
-		Scopes:      []string{"user:email"},
-		InfoURL:     fmt.Sprintf("http://localhost:%d/user", authPort),
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:      "mock_" + data.value("id"),
-				Name:    data.value("name"),
-				Picture: data.value("picture"),
-			}
-			return userInfo
-		},
-	}
-
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret",
-		JwtService: NewJWT("12345", false, time.Hour), Admins: []string{"mock_myuser2"},
-		// AvatarProxy:  &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"},
-		IsVerifiedFn: func(siteID, userID string) bool { return userID == "mock_myuser2" }}
-	provider = initProvider(params, provider)
-
-	ts := &http.Server{Addr: fmt.Sprintf(":%d", loginPort), Handler: provider.Routes()}
-
-	count := 0
-	useIds := []string{"myuser1", "myuser2"}
-
-	oauth := &http.Server{
-		Addr: fmt.Sprintf(":%d", authPort),
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			log.Printf("[MOCK OAUTH] request %s %s %+v", r.Method, r.URL, r.Header)
-			switch {
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
-				state := r.URL.Query().Get("state")
-				w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s",
-					loginPort, state))
-				w.WriteHeader(302)
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"):
-				res := `{
-					"access_token":"MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3",
-					"token_type":"bearer",
-					"expires_in":3600,
-					"refresh_token":"IwOGYzYTlmM2YxOTQ5MGE3YmNmMDFkNTVk",
-					"scope":"create",
-					"state":"12345678"
-					}`
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			case strings.HasPrefix(r.URL.Path, "/user"):
-				res := fmt.Sprintf(`{
-					"id": "%s",
-					"name":"blah",
-					"picture":"http://exmple.com/pic1.png"
-					}`, useIds[count])
-				count++
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			default:
-				t.Fatalf("unexpected oauth request %s %s", r.Method, r.URL)
-			}
-		}),
-	}
-
-	go func() { _ = oauth.ListenAndServe() }()
-	go func() { _ = ts.ListenAndServe() }()
-
-	time.Sleep(time.Millisecond * 100) // let them start
-	return ts, oauth
-}

app/rest/auth/providers_test.go

@@ -1,83 +0,0 @@
-package auth
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestProviders_NewGoogle(t *testing.T) {
-	r := NewGoogle(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "google", r.Name)
-
-	udata := userData{"sub": "1234567890", "name": "test user", "picture": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// no name in data
-	udata = userData{"sub": "1234567890", "picture": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "noname_1b30", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewGithub(t *testing.T) {
-	r := NewGithub(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "github", r.Name)
-
-	udata := userData{"login": "lll", "name": "test user", "avatar_url": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// nil name in data (json response contains `"name": null`); using login, it's always required
-	udata = userData{"login": "lll", "name": nil, "avatar_url": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "lll", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewFacebook(t *testing.T) {
-	r := NewFacebook(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "facebook", r.Name)
-
-	udata := userData{"id": "myid", "name": "test user"}
-	user := r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "test user", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	udata = userData{"id": "myid", "name": ""}
-	user = r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "facebook_6e34471", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewYandex(t *testing.T) {
-	r := NewYandex(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "yandex", r.Name)
-
-	udata := userData{"id": "1234567890", "display_name": "Vasya P", "default_avatar_id": "131652443"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya P", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "https://avatars.yandex.net/get-yapic/131652443/islands-200", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "display_name": null, "default_avatar_id": null
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": nil, "real_name": "Vasya Pupkin", "default_avatar_id": nil}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya Pupkin", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// empty "display_name", empty "default_avatar_id", empty "real_name"
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": "", "real_name": "", "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "real_name": null
-	udata = userData{"id": "1234567890", "login": "vasya", "real_name": nil, "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}

app/rest/cache/cache.go

@@ -1,45 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
-)
-
-// LoadingCache defines interface for caching
-type LoadingCache interface {
-	Get(key string, fn func() ([]byte, error)) (data []byte, err error)
-	Flush(scopes ...string)
-}
-
-// Key makes full key from primary key and scopes
-func Key(key string, scopes ...string) string {
-	return strings.Join(scopes, "$$") + "@@" + key
-}
-
-// ParseKey gets compound key created by Key func and split it to the actual key and scopes
-func ParseKey(fullKey string) (key string, scopes []string, err error) {
-	elems := strings.Split(fullKey, "@@")
-	if len(elems) != 2 {
-		return "", nil, errors.Errorf("can't parse cache key %s", key)
-	}
-	scopes = strings.Split(elems[0], "$$")
-	if len(scopes) == 1 && scopes[0] == "" {
-		scopes = []string{}
-	}
-	key = elems[1]
-	return key, scopes, nil
-}
-
-// URLKey gets url from request to use it as cache key
-// admins will have different keys in order to prevent leak of admin-only data to regular users
-func URLKey(r *http.Request) string {
-	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
-		key = adminPrefix + key
-	}
-	return key
-}

app/rest/cache/cache_test.go

@@ -1,54 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestCache_Keys(t *testing.T) {
-	tbl := []struct {
-		key    string
-		scopes []string
-		full   string
-	}{
-		{"key1", []string{"s1"}, "s1@@key1"},
-		{"key2", []string{"s11", "s2"}, "s11$$s2@@key2"},
-		{"key3", []string{}, "@@key3"},
-	}
-
-	for n, tt := range tbl {
-		full := Key(tt.key, tt.scopes...)
-		assert.Equal(t, tt.full, full, "making key, #%d", n)
-
-		k, s, e := ParseKey(full)
-		assert.Nil(t, e)
-		assert.Equal(t, tt.scopes, s)
-		assert.Equal(t, tt.key, k)
-	}
-
-	_, _, err := ParseKey("abc")
-	assert.Error(t, err)
-	_, _, err = ParseKey("")
-	assert.Error(t, err)
-}
-
-func TestCache_URLKey(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah/123", nil)
-	assert.Nil(t, err)
-	key := URLKey(r)
-	assert.Equal(t, "http://blah/123", key)
-
-	r, err = http.NewRequest("GET", "http://blah/123?key=v&k2=v2", nil)
-	assert.Nil(t, err)
-	key = URLKey(r)
-	assert.Equal(t, "http://blah/123?key=v&k2=v2", key)
-
-	user := store.User{Admin: true}
-	r = rest.SetUserInfo(r, user)
-	key = URLKey(r)
-	assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key)
-}

app/rest/cache/memory_test.go

@@ -1,310 +0,0 @@
-package cache
-
-import (
-	"fmt"
-	"math/rand"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMemoryCache_Get(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }))
-	require.Nil(t, err)
-	res, err := lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	res, err = lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	lc.Flush()
-	time.Sleep(100 * time.Millisecond) // let postFn to do its thing
-	assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall))
-
-	_, err = lc.Get("key", func() ([]byte, error) {
-		return nil, errors.New("err")
-	})
-	assert.NotNil(t, err)
-}
-
-func TestMemoryCache_MaxKeys(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }),
-		MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-
-	// put 5 keys to cache
-	for i := 0; i < 5; i++ {
-		res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-			atomic.AddInt32(&coldCalls, 1)
-			return []byte(fmt.Sprintf("result-%d", i)), nil
-		})
-		assert.Nil(t, e)
-		assert.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-		assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls))
-		assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-	}
-
-	// check if really cached
-	res, err := lc.Get("key-3", func() ([]byte, error) {
-		return []byte("result-blah"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-3", string(res), "should be cached")
-
-	// try to cache after maxKeys reached
-	res, err = lc.Get("key-X", func() ([]byte, error) {
-		return []byte("result-X"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-X", string(res))
-
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-
-	// put to cache and make sure it cached
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxValueSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-
-	// put too big value to cache and make sure it is not cached
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("1234567890"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "1234567890", string(res))
-
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("result-big"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-big", string(res), "got not cached value")
-}
-
-func TestMemoryCache_MaxCacheSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(50), MaxCacheSize(20))
-	require.Nil(t, err)
-
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-	assert.Equal(t, int64(8), lc.(*memoryCache).currentSize)
-
-	_, err = lc.Get("key-Z2", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	// this will cause removal
-	_, err = lc.Get("key-Z3", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) {
-	lc, err := NewMemoryCache(MaxCacheSize(123), MaxKeys(10000))
-	require.Nil(t, err)
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond)
-			defer wg.Done()
-			res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-			size := atomic.LoadInt64(&lc.(*memoryCache).currentSize)
-			require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel
-		}()
-	}
-	wg.Wait()
-	assert.True(t, lc.(*memoryCache).currentSize < 123 && lc.(*memoryCache).currentSize >= 0)
-	t.Log("size=", lc.(*memoryCache).currentSize)
-}
-
-func TestMemoryCache_Parallel(t *testing.T) {
-	var coldCalls int32
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get("key", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			defer wg.Done()
-			res, err := lc.Get("key", func() ([]byte, error) {
-				atomic.AddInt32(&coldCalls, 1)
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, "value", string(res))
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls))
-}
-
-func TestMemoryCache_Scopes(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		return []byte("value2"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value2", string(res))
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-	lc.Flush("s1")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	_, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		assert.Fail(t, "should stay")
-		return nil, nil
-	})
-	assert.Nil(t, err)
-	res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value-upd"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value-upd", string(res), "was deleted, update")
-}
-
-func TestMemoryCache_Flush(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	addToCache := func(key string, scopes ...string) {
-		res, err := lc.Get(key, func() ([]byte, error) {
-			return []byte("value" + key), nil
-		})
-		require.Nil(t, err)
-		require.Equal(t, "value"+key, string(res))
-	}
-
-	init := func() {
-		lc.Flush()
-		addToCache(Key("key1", "s1", "s2"))
-		addToCache(Key("key2", "s1", "s2", "s3"))
-		addToCache(Key("key3", "s1", "s2", "s3"))
-		addToCache(Key("key4", "s2", "s3"))
-		addToCache(Key("key5", "s2"))
-		addToCache(Key("key6"))
-		addToCache(Key("key7", "s4", "s3"))
-		require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init")
-	}
-
-	tbl := []struct {
-		scopes []string
-		left   int
-		msg    string
-	}{
-		{[]string{}, 0, "full flush, no scopes"},
-		{[]string{"s0"}, 7, "flush wrong scope"},
-		{[]string{"s1"}, 4, "flush s1 scope"},
-		{[]string{"s2", "s1"}, 2, "flush s2+s1 scope"},
-		{[]string{"s1", "s2"}, 2, "flush s1+s2 scope"},
-		{[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"},
-		{[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"},
-		{[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"},
-	}
-
-	for i, tt := range tbl {
-		init()
-		lc.Flush(tt.scopes...)
-		assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i)
-	}
-}
-
-func TestMemoryCache_FlushFailed(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-	val, err := lc.Get("invalid-composite", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(val))
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	lc.Flush("invalid-composite")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_BadOptions(t *testing.T) {
-	_, err := NewMemoryCache(MaxCacheSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size or MaxCacheSize, -1")
-
-	_, err = NewMemoryCache(MaxKeys(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxKeys, -1")
-
-	_, err = NewMemoryCache(MaxValSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxValSize, -1")
-}

app/rest/proxy/avatar.go

@@ -1,119 +0,0 @@
-package proxy
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Avatar provides file-system store and http handler for avatars
-// On user login auth will call Put and it will retrieve and save picture locally.
-type Avatar struct {
-	Store     AvatarStore
-	RoutePath string
-	RemarkURL string
-}
-
-const imgSfx = ".image"
-
-// Put stores retrieved avatar to StorePath. Gets image from user info. Returns proxied url
-func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
-
-	// no picture for user, try default avatar
-	if u.Picture == "" {
-		return "", errors.Errorf("no picture for %s", u.ID)
-	}
-
-	// load avatar from remote location
-	client := http.Client{Timeout: 10 * time.Second}
-	var resp *http.Response
-	err = retry(5, time.Second, func() error {
-		var e error
-		resp, e = client.Get(u.Picture)
-		return e
-	})
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch avatar from the orig")
-	}
-
-	defer func() {
-		if e := resp.Body.Close(); e != nil {
-			log.Printf("[WARN] can't close response body, %s", e)
-		}
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status)
-	}
-
-	avatar, err := p.Store.Put(u.ID, resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatar, u.Name)
-	return p.RemarkURL + p.RoutePath + "/" + avatar, nil
-}
-
-// Routes returns auth routes for given provider
-func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string, chi.Router) {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-
-	// GET /123456789.image
-	router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) {
-
-		avatar := chi.URLParam(r, "avatar")
-
-		// enforce client-side caching
-		etag := `"` + avatar + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-
-		avReader, size, err := p.Store.Get(avatar)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar")
-			return
-		}
-
-		defer func() {
-			if e := avReader.Close(); e != nil {
-				log.Printf("[WARN] can't close avatar reader for %s, %s", avatar, e)
-			}
-		}()
-
-		w.Header().Set("Content-Type", "image/*")
-		w.Header().Set("Content-Length", strconv.Itoa(size))
-		w.WriteHeader(http.StatusOK)
-		if _, err = io.Copy(w, avReader); err != nil {
-			log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
-		}
-	})
-
-	return p.RoutePath, router
-}
-
-func retry(retries int, delay time.Duration, fn func() error) (err error) {
-	for i := 0; i < retries; i++ {
-		if err = fn(); err == nil {
-			return nil
-		}
-		time.Sleep(delay)
-	}
-	return errors.Wrap(err, "retry failed")
-}

app/rest/proxy/avatar_store.go

@@ -1,145 +0,0 @@
-package proxy
-
-//go:generate sh -c "mockery -inpkg -name AvatarStore -print > /tmp/mock.tmp && mv /tmp/mock.tmp avatar_store_mock.go"
-
-import (
-	"bytes"
-	"fmt"
-	"hash/crc64"
-	"image"
-	"image/png"
-	"io"
-	"log"
-	"os"
-	"path"
-	"strings"
-	"sync"
-
-	// Initializing packages for supporting GIF and JPEG formats.
-	_ "image/gif"
-	_ "image/jpeg"
-
-	"github.com/pkg/errors"
-	"golang.org/x/image/draw"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// AvatarStore defines interface to store and serve avatars
-type AvatarStore interface {
-	Put(userID string, reader io.Reader) (avatar string, err error)
-	Get(avatar string) (reader io.ReadCloser, size int, err error)
-}
-
-// FSAvatarStore implements AvatarStore for local file system
-type FSAvatarStore struct {
-	storePath   string
-	resizeLimit int
-	ctcTable    *crc64.Table
-	once        sync.Once
-}
-
-// NewFSAvatarStore makes file-system avatar store
-func NewFSAvatarStore(storePath string, resizeLimit int) *FSAvatarStore {
-	return &FSAvatarStore{storePath: storePath, resizeLimit: resizeLimit}
-}
-
-// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
-func (fs *FSAvatarStore) Put(userID string, reader io.Reader) (avatar string, err error) {
-	id := store.EncodeID(userID)
-	location := fs.location(id) // location adds partition to path
-
-	if _, err = os.Stat(location); os.IsNotExist(err) {
-		if e := os.Mkdir(location, 0700); e != nil {
-			return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
-		}
-	}
-
-	avFile := path.Join(location, id+imgSfx)
-	fh, err := os.Create(avFile)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't create file %s", avFile)
-	}
-	defer func() {
-		if e := fh.Close(); e != nil {
-			log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
-		}
-	}()
-
-	// Trying to resize avatar.
-	if reader = resize(reader, fs.resizeLimit); reader == nil {
-		return "", errors.New("avatar reader is nil")
-	}
-
-	if _, err = io.Copy(fh, reader); err != nil {
-		return "", errors.Wrapf(err, "can't save file %s", avFile)
-	}
-	return id + imgSfx, nil
-}
-
-// Get avatar reader for avatar id.image
-func (fs *FSAvatarStore) Get(avatar string) (reader io.ReadCloser, size int, err error) {
-	location := fs.location(strings.TrimSuffix(avatar, imgSfx))
-	avFile := path.Join(location, avatar)
-	fh, err := os.Open(avFile)
-	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't load avatar %s, id")
-	}
-	if fi, e := fh.Stat(); e == nil {
-		size = int(fi.Size())
-	}
-	return fh, size, nil
-}
-
-// get location (directory) for user id by adding partition to final path in order to keep files
-// in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/avatars.test/92
-func (fs *FSAvatarStore) location(id string) string {
-	fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
-	checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
-	partition := checksum64 % 100
-	return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
-}
-
-// Resizes an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
-// (width or height) preserving aspect ratio.
-// Returns original reader if resizing is not needed or failed.
-func resize(reader io.Reader, limit int) io.Reader {
-	if reader == nil {
-		log.Print("[WARN] avatar resize(): reader is nil")
-		return nil
-	}
-	if limit <= 0 {
-		log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
-		return reader
-	}
-
-	var teeBuf bytes.Buffer
-	tee := io.TeeReader(reader, &teeBuf)
-	src, _, err := image.Decode(tee)
-	if err != nil {
-		log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
-		return &teeBuf
-	}
-
-	bounds := src.Bounds()
-	w, h := bounds.Dx(), bounds.Dy()
-	if w <= limit && h <= limit || w <= 0 || h <= 0 {
-		log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
-		return &teeBuf
-	}
-	newW, newH := w*limit/h, limit
-	if w > h {
-		newW, newH = limit, h*limit/w
-	}
-	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
-	// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
-	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
-
-	var out bytes.Buffer
-	if err = png.Encode(&out, m); err != nil {
-		log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
-		return &teeBuf
-	}
-	return &out
-}

app/rest/proxy/avatar_store_test.go

@@ -1,135 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"image"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAvatarStore_Put(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	avatar, err := p.Put("user1", nil)
-	assert.Equal(t, "", avatar)
-	assert.EqualError(t, err, "avatar reader is nil")
-
-	avatar, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	require.Nil(t, err)
-	assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	avatar, err = p.Put("user2", strings.NewReader("some picture bin data 123"))
-	require.Nil(t, err)
-	assert.Equal(t, "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(25), fi.Size())
-
-	// with resize
-	file, e := os.Open("testdata/circles.png")
-	require.Nil(t, e)
-	avatar, err = p.Put("user3", file)
-	require.Nil(t, err)
-	assert.Equal(t, "0b7f849446d3383546d15a480966084442cd2193.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/60/0b7f849446d3383546d15a480966084442cd2193.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(6986), fi.Size())
-
-	p = NewFSAvatarStore("/dev/null", 300)
-	_, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
-}
-
-func TestAvatarStore_Get(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test/30", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-	err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
-	assert.Nil(t, err)
-	r, size, err := p.Get("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.Nil(t, err)
-	assert.Equal(t, 9, size)
-	data, err := ioutil.ReadAll(r)
-	assert.Nil(t, err)
-	assert.Equal(t, "something", string(data))
-}
-
-func TestAvatarStore_Location(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-
-	tbl := []struct {
-		id  string
-		res string
-	}{
-		{"abc", "/tmp/avatars.test/35"},
-		{"xyz", "/tmp/avatars.test/69"},
-		{"blah blah", "/tmp/avatars.test/29"},
-	}
-
-	for i, tt := range tbl {
-		assert.Equal(t, tt.res, p.location(tt.id), "test #%d", i)
-	}
-}
-
-func TestAvatarStore_resize(t *testing.T) {
-	checkC := func(t *testing.T, r io.Reader, cExp []byte) {
-		content, err := ioutil.ReadAll(r)
-		require.NoError(t, err)
-		assert.Equal(t, cExp, content)
-	}
-
-	// Reader is nil.
-	resizedR := resize(nil, 100)
-	// assert.EqualError(t, err, "limit should be greater than 0")
-	assert.Nil(t, resizedR)
-
-	// Negative limit error.
-	resizedR = resize(strings.NewReader("some picture bin data"), -1)
-	require.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("some picture bin data"))
-
-	// Decode error.
-	resizedR = resize(strings.NewReader("invalid image content"), 100)
-	assert.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("invalid image content"))
-
-	cases := []struct {
-		file   string
-		wr, hr int
-	}{
-		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
-		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
-	}
-
-	for _, c := range cases {
-		img, err := ioutil.ReadFile(c.file)
-		require.Nil(t, err, "can't open test file %s", c.file)
-
-		// No need for resize, avatar dimensions are smaller than resize limit.
-		resizedR = resize(bytes.NewReader(img), 800)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-		checkC(t, resizedR, img)
-
-		// Resizing to half of width. Check resizedR avatar format PNG.
-		resizedR = resize(bytes.NewReader(img), 400)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-
-		imgRz, format, err := image.Decode(resizedR)
-		assert.Nilf(t, err, "file %s", c.file)
-		assert.Equalf(t, "png", format, "file %s", c.file)
-		bounds := imgRz.Bounds()
-		assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
-		assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
-	}
-}

app/rest/proxy/avatar_test.go

@@ -1,142 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestAvatar_Put(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	res, err := p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/b3daa77b4c04a9551b8781d03191fe098f325e67.image", res)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	u.ID = "user2"
-	res, err = p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", res)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-}
-
-func TestAvatar_PutFailed(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		log.Print("request: ", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-
-	u := store.User{ID: "user1", Name: "user1 name"}
-	_, err := p.Put(u)
-	assert.EqualError(t, err, "no picture for user1")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "connect: connection refused")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/avatar/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "failed to get avatar from the orig")
-}
-
-func TestAvatar_Routes(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			w.Header().Set("Custom-Header", "xyz")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	_, err := p.Put(u)
-	assert.NoError(t, err)
-
-	req, err := http.NewRequest("GET", "/b3daa77b4c04a9551b8781d03191fe098f325e67.image", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	rr := httptest.NewRecorder()
-	_, routes := p.Routes()
-	handler := http.Handler(routes)
-	handler.ServeHTTP(rr, req)
-
-	assert.Equal(t, http.StatusOK, rr.Code)
-
-	assert.Equal(t, []string{"image/*"}, rr.HeaderMap["Content-Type"])
-	assert.Equal(t, []string{"21"}, rr.HeaderMap["Content-Length"])
-	assert.Equal(t, []string(nil), rr.HeaderMap["Custom-Header"], "strip all custom headers")
-	assert.NotNil(t, rr.HeaderMap["Etag"])
-
-	bb := bytes.Buffer{}
-	sz, err := io.Copy(&bb, rr.Body)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), sz)
-	assert.Equal(t, "some picture bin data", bb.String())
-}
-
-func TestAvatar_Retry(t *testing.T) {
-	i := 0
-	err := retry(5, time.Millisecond, func() error {
-		if i == 3 {
-			return nil
-		}
-		i++
-		return errors.New("err")
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 3, i)
-
-	st := time.Now()
-	err = retry(5, time.Millisecond, func() error {
-		return errors.New("err")
-	})
-	assert.NotNil(t, err)
-	assert.True(t, time.Since(st) >= time.Microsecond*5)
-}

app/rest/tree_test.go

@@ -1,396 +0,0 @@
-package rest
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestMakeTree(t *testing.T) {
-
-	loc := store.Locator{URL: "url", SiteID: "site"}
-	ts := func(min int, sec int) time.Time { return time.Date(2017, 12, 25, 19, min, sec, 0, time.UTC) }
-
-	// unsorted by purpose
-	comments := []store.Comment{
-		{Locator: loc, ID: "14", ParentID: "1", Timestamp: ts(46, 14)},
-		{Locator: loc, ID: "1", Timestamp: ts(46, 1)},
-		{Locator: loc, ID: "2", Timestamp: ts(47, 2)},
-		{Locator: loc, ID: "11", ParentID: "1", Timestamp: ts(46, 11)},
-		{Locator: loc, ID: "13", ParentID: "1", Timestamp: ts(46, 13)},
-		{Locator: loc, ID: "12", ParentID: "1", Timestamp: ts(46, 12)},
-		{Locator: loc, ID: "131", ParentID: "13", Timestamp: ts(46, 31)},
-		{Locator: loc, ID: "132", ParentID: "13", Timestamp: ts(46, 32)},
-		{Locator: loc, ID: "21", ParentID: "2", Timestamp: ts(47, 21)},
-		{Locator: loc, ID: "22", ParentID: "2", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "4", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "3", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "5", Deleted: true},
-		{Locator: loc, ID: "6", Deleted: true},
-		{Locator: loc, ID: "61", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "62", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
-	}
-
-	res := MakeTree(comments, "time", 0)
-
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(res)
-	assert.Nil(t, err)
-	expected, actual := cleanFormatting(expJSON, buf.String())
-	assert.Equal(t, expected, actual)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
-
-	res = MakeTree([]store.Comment{}, "time", 0)
-	assert.Equal(t, &Tree{}, res)
-
-	res = MakeTree(comments, "time", 10)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
-}
-
-func TestTreeSortNodes(t *testing.T) {
-	// unsorted by purpose
-	comments := []store.Comment{
-		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
-		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
-		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
-		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
-		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
-		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
-		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
-		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
-		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
-	}
-
-	res := MakeTree(comments, "+active", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-
-	res = MakeTree(comments, "-active", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "+time", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-time", 0)
-	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "+score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-score", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "undefined", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-}
-
-func BenchmarkTree(b *testing.B) {
-	comments := []store.Comment{}
-	data, err := ioutil.ReadFile("testfile.json")
-	assert.Nil(b, err)
-	err = json.Unmarshal(data, &comments)
-	assert.Nil(b, err)
-
-	for i := 0; i < b.N; i++ {
-		res := MakeTree(comments, "time", 0)
-		assert.NotNil(b, res)
-	}
-}
-
-const expJSON = `{
-  "comments": [
-    {
-      "comment": {
-        "id": "1",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": { 
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:46:01Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "11",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-          	  "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:11Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "12",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:12Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "13",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:13Z"
-          },
-          "replies": [
-            {
-              "comment": {
-                "id": "131",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:31Z"
-              }
-            },
-            {
-              "comment": {
-                "id": "132",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:32Z"
-              }
-            }
-          ]
-        },
-        {
-          "comment": {
-            "id": "14",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:14Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "2",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:02Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "21",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:21Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "22",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:22Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "4",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    },
-    {
-      "comment": {
-        "id": "3",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    }
-  ],
-  "info": {
-	 "url": "url",
-	 "count": 12,
-     "first_time": "2017-12-25T19:46:01Z",
-     "last_time": "2017-12-25T19:47:22Z"
-   }
-}
-`
-
-func cleanFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/user.go

@@ -1,32 +0,0 @@
-package rest
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/umputun/remark/app/store"
-)
-
-type contextKey string
-
-// GetUserInfo returns user from request context
-func GetUserInfo(r *http.Request) (user store.User, err error) {
-
-	ctx := r.Context()
-	if ctx == nil {
-		return store.User{}, errors.New("no info about user")
-	}
-	if u, ok := ctx.Value(contextKey("user")).(store.User); ok {
-		return u, nil
-	}
-
-	return store.User{}, errors.New("user can't be parsed")
-}
-
-// SetUserInfo sets user into request context
-func SetUserInfo(r *http.Request, user store.User) *http.Request {
-	ctx := r.Context()
-	ctx = context.WithValue(ctx, contextKey("user"), user)
-	return r.WithContext(ctx)
-}

app/rest/user_test.go

@@ -1,21 +0,0 @@
-package rest
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestGetUserInfo(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
-	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
-
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
-	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
-}

app/store/service/service.go

@@ -1,189 +0,0 @@
-package service
-
-import (
-	"sync"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-)
-
-// DataStore wraps store.Interface with additional methods
-type DataStore struct {
-	engine.Interface
-	EditDuration   time.Duration
-	Secret         string
-	MaxCommentSize int
-
-	// granular locks
-	scopedLocks struct {
-		sync.Mutex
-		sync.Once
-		locks map[string]sync.Locker
-	}
-}
-
-const defaultCommentMaxSize = 2000
-
-// Create prepares comment and forward to Interface.Create
-func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
-	// fill ID and time if empty
-	if comment.ID == "" {
-		comment.ID = uuid.New().String()
-	}
-	if comment.Timestamp.IsZero() {
-		comment.Timestamp = time.Now()
-	}
-	// reset votes if nothing
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-
-	comment.Sanitize()            // clear potentially dangerous js from all parts of comment
-	comment.User.HashIP(s.Secret) // replace ip by hash
-
-	return s.Interface.Create(comment)
-}
-
-// SetPin pin/un-pin comment as special
-func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
-	comment, err := s.Get(locator, commentID)
-	if err != nil {
-		return err
-	}
-	comment.Pin = status
-	return s.Put(locator, comment)
-}
-
-// Vote for comment by id and locator
-func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
-
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
-	defer cLock.Unlock()
-
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	if comment.User.ID == userID && userID != "dev" {
-		return comment, errors.Errorf("user %s can not vote for his own comment %s", userID, commentID)
-	}
-
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-	v, voted := comment.Votes[userID]
-
-	if voted && v == val {
-		return comment, errors.Errorf("user %s already voted for %s", userID, commentID)
-	}
-
-	// reset vote if user changed to opposite
-	if voted && v != val {
-		delete(comment.Votes, userID)
-	}
-
-	// add to voted map if first vote
-	if !voted {
-		comment.Votes[userID] = val
-	}
-
-	// update score
-	if val {
-		comment.Score++
-	} else {
-		comment.Score--
-	}
-
-	return comment, s.Put(locator, comment)
-}
-
-// EditRequest contains fields needed for comment update
-type EditRequest struct {
-	Text    string
-	Orig    string
-	Summary string
-}
-
-// EditComment to edit text and update Edit info
-func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	// edit allowed in editDuration window only
-	if s.EditDuration > 0 && time.Now().After(comment.Timestamp.Add(s.EditDuration)) {
-		return comment, errors.Errorf("too late to edit %s", commentID)
-	}
-
-	comment.Text = req.Text
-	comment.Orig = req.Orig
-	comment.Edit = &store.Edit{
-		Timestamp: time.Now(),
-		Summary:   req.Summary,
-	}
-
-	comment.Sanitize()
-	err = s.Put(locator, comment)
-	return comment, err
-}
-
-// Counts returns postID+count list for given comments
-func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
-	res := []store.PostInfo{}
-	for _, p := range postIDs {
-		if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
-			res = append(res, store.PostInfo{URL: p, Count: c})
-		}
-	}
-	return res, nil
-}
-
-// ValidateComment checks if comment size below max and user fields set
-func (s *DataStore) ValidateComment(c *store.Comment) error {
-	maxSize := s.MaxCommentSize
-	if s.MaxCommentSize <= 0 {
-		maxSize = defaultCommentMaxSize
-	}
-	if c.Orig == "" {
-		return errors.New("empty comment text")
-	}
-	if len([]rune(c.Orig)) > maxSize {
-		return errors.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, len([]rune(c.Orig)))
-	}
-	if c.User.ID == "" || c.User.Name == "" {
-		return errors.Errorf("empty user info")
-	}
-	return nil
-}
-
-// IsVerifiedFn returns func to check if user verified or not
-func (s *DataStore) IsVerifiedFn() func(siteID string, userID string) bool {
-	return func(siteID string, userID string) bool {
-		if siteID == "" {
-			return false
-		}
-		return s.IsVerified(siteID, userID)
-	}
-}
-
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
-	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
-
-	s.scopedLocks.Lock()
-	lock, ok := s.scopedLocks.locks[id]
-	if !ok {
-		lock = &sync.Mutex{}
-		s.scopedLocks.locks[id] = lock
-	}
-	s.scopedLocks.Unlock()
-
-	return lock
-}

backend/Gopkg.lock

@@ -0,0 +1,429 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:180876db3ec295bb9f0babec5ca926fe9f2036b747b7c5bfcd13b333023e7cfd"
+  name = "cloud.google.com/go"
+  packages = ["compute/metadata"]
+  pruneopts = "UT"
+  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
+  version = "v0.18.0"
+
+[[projects]]
+  digest = "1:bff7b2530f02b143623e260c11df5cbf34e0faeaca6aa001a8be31f333518ca9"
+  name = "github.com/PuerkitoBio/goquery"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
+  version = "v1.4.0"
+
+[[projects]]
+  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
+  name = "github.com/andybalholm/cascadia"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "901648c87902174f774fac311d7f176f8647bdaa"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e"
+  name = "github.com/coreos/bbolt"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
+  version = "v1.3.0"
+
+[[projects]]
+  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  pruneopts = "UT"
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
+
+[[projects]]
+  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
+  name = "github.com/dgrijalva/jwt-go"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
+  version = "v3.2.0"
+
+[[projects]]
+  digest = "1:82c6357bc57f8417f993d490f6c07a9f0b5682ac68b1a64b93a189dece7c5bf5"
+  name = "github.com/didip/tollbooth"
+  packages = [
+    ".",
+    "errors",
+    "libstring",
+    "limiter",
+  ]
+  pruneopts = "UT"
+  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
+  version = "v4.0.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:60fb125752a234a0a43bfc281bfdd9726fd1071a13f66bb35ec4b8e7ed1ef642"
+  name = "github.com/didip/tollbooth_chi"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
+
+[[projects]]
+  digest = "1:4b08116de0de75c041bb341686f0b139930f26cb84dfdf7641d435548114181d"
+  name = "github.com/globalsign/mgo"
+  packages = [
+    ".",
+    "bson",
+    "internal/json",
+    "internal/sasl",
+    "internal/scram",
+  ]
+  pruneopts = "UT"
+  revision = "113d3961e7311526535a1ef7042196563d442761"
+  version = "r2018.06.15"
+
+[[projects]]
+  digest = "1:4eda9f7bf70f5145b3b9ed3f18ac93e9b1a0e38906eb69e526380c34861e2b07"
+  name = "github.com/go-chi/chi"
+  packages = [
+    ".",
+    "middleware",
+  ]
+  pruneopts = "UT"
+  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
+  version = "v3.3.2"
+
+[[projects]]
+  digest = "1:dfa416a1bb8139f30832543340f972f65c0db9932034cb6a1b42c5ac615a3fb8"
+  name = "github.com/go-chi/cors"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dba6525398619dead495962a916728e7ee2ca322"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:75f324f9a6b76bca2fdd087ba169de30bc28a95a7139a6cefd5a9ac7582f6dab"
+  name = "github.com/go-chi/render"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:38940db39a8ee330d3e7f68ab4c298575462f5986ae2bb0cafa86f095e9d32ad"
+  name = "github.com/go-pkgz/auth"
+  packages = [
+    ".",
+    "avatar",
+    "logger",
+    "middleware",
+    "provider",
+    "token",
+  ]
+  pruneopts = "UT"
+  revision = "7597c083287c33ba5362687a7815a032fd92c418"
+  version = "v0.4.1"
+
+[[projects]]
+  digest = "1:7b1f422f560b103f435f8501f854f238f1ce00d0f8146e76eacdfa71f2c8d8c0"
+  name = "github.com/go-pkgz/lgr"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dbd940fec9650260d15aa071cab658d0502ed92f"
+  version = "v0.3.2"
+
+[[projects]]
+  digest = "1:c509e3f646c48148f2239fa66d5cd62785dfd057b5d2b46972e45b60049c19ed"
+  name = "github.com/go-pkgz/mongo"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ee1e9a167094f8cdbc27407907aa694fff350ec3"
+  version = "v1.1.0"
+
+[[projects]]
+  digest = "1:0f24c72d7e9bcb682b907be0461ac552973cd4b3f1b60b04b725f6d74a3e59e7"
+  name = "github.com/go-pkgz/repeater"
+  packages = [
+    ".",
+    "strategy",
+  ]
+  pruneopts = "UT"
+  revision = "f2a67dcf050cab24d57132a7d8b45553ceab817b"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:9aba5c95373481f118e57e9740d9e82f86802d86849809ab89324bcb4f236451"
+  name = "github.com/go-pkgz/rest"
+  packages = [
+    ".",
+    "cache",
+    "logger",
+  ]
+  pruneopts = "UT"
+  revision = "e7d08d0194d613b8854de2e487bf7732500fa153"
+  version = "v1.2.0"
+
+[[projects]]
+  digest = "1:ffc060c551980d37ee9e428ef528ee2813137249ccebb0bfc412ef83071cac91"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  pruneopts = "UT"
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b"
+  name = "github.com/google/uuid"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
+  version = "0.2"
+
+[[projects]]
+  digest = "1:2b418e5e28a68ccab236a22f344140cebab2d90c3a4a3f5593ecbb82cfe0e5ce"
+  name = "github.com/gorilla/feeds"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
+  version = "v1.1.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:07671f8997086ed115824d1974507d2b147d1e0463675ea5dbf3be89b1c2c563"
+  name = "github.com/hashicorp/errwrap"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
+
+[[projects]]
+  branch = "master"
+  digest = "1:e5048c5da80697be2fcdecc944e29d2999e01fd7f48b643168443209779f3463"
+  name = "github.com/hashicorp/go-multierror"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
+
+[[projects]]
+  branch = "master"
+  digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db"
+  name = "github.com/hashicorp/golang-lru"
+  packages = [
+    ".",
+    "simplelru",
+  ]
+  pruneopts = "UT"
+  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:3217800110ab50cd0e0784307be46b5344c0c103dbd15a16d0994ae4abdc96ab"
+  name = "github.com/jessevdk/go-flags"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
+
+[[projects]]
+  branch = "master"
+  digest = "1:e83a8cf54ecc5c4efdbc88aa914578773d4d6897470b698ae315b5734081e8ed"
+  name = "github.com/microcosm-cc/bluemonday"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
+
+[[projects]]
+  branch = "master"
+  digest = "1:ea014b8bb16b0decc3393baeafc3b19815bcaf92329fe643eef5c0aa89bd3291"
+  name = "github.com/nullrocks/identicon"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "7875f45b0022edded6377e40639d8aa620193a62"
+
+[[projects]]
+  digest = "1:808cdddf087fb64baeae67b8dfaee2069034d9704923a3cb8bd96a995421a625"
+  name = "github.com/patrickmn/go-cache"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
+  version = "v2.1.0"
+
+[[projects]]
+  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  pruneopts = "UT"
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:bc91590d3e20673d5e33267fc140e7dadddde0b84f2e9030547ba86859d2d13e"
+  name = "github.com/rakyll/statik"
+  packages = ["fs"]
+  pruneopts = "UT"
+  revision = "19b88da8fc15428620782ba18f68423130e7ac7d"
+  version = "v0.1.3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:def689e73e9252f6f7fe66834a76751a41b767e03daab299e607e7226c58a855"
+  name = "github.com/shurcooL/sanitized_anchor_name"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
+
+[[projects]]
+  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
+  name = "github.com/stretchr/testify"
+  packages = [
+    "assert",
+    "require",
+  ]
+  pruneopts = "UT"
+  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
+  version = "v1.2.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:674a2405f77863f4f362e3e45078fd9caa647dc1c0e8b022e3b9dc1ae89fbc8f"
+  name = "golang.org/x/crypto"
+  packages = [
+    "acme",
+    "acme/autocert",
+  ]
+  pruneopts = "UT"
+  revision = "4d3f4d9ffa16a13f451c3b2999e9c49e9750bf06"
+
+[[projects]]
+  branch = "master"
+  digest = "1:b34062e39d8f3172fdd0c5c22ca1a3badeb2ddde295a997b0b63441e96d916f7"
+  name = "golang.org/x/image"
+  packages = [
+    "draw",
+    "math/f64",
+  ]
+  pruneopts = "UT"
+  revision = "af66defab954cb421ca110193eed9477c8541e2a"
+
+[[projects]]
+  branch = "master"
+  digest = "1:ac7eaa5f1179480f517d32831225215cc20940152d66be29f3d5204ea15d425f"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "context/ctxhttp",
+    "html",
+    "html/atom",
+  ]
+  pruneopts = "UT"
+  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
+
+[[projects]]
+  branch = "master"
+  digest = "1:ccb0526e2eb5d454a25a536634fade769664eaa93ad2e4cd4107967bbc01b4e8"
+  name = "golang.org/x/oauth2"
+  packages = [
+    ".",
+    "facebook",
+    "github",
+    "google",
+    "internal",
+    "jws",
+    "jwt",
+    "yandex",
+  ]
+  pruneopts = "UT"
+  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
+
+[[projects]]
+  branch = "master"
+  digest = "1:ba7d5e85e8b4f084fae02a1a9d7462980e889d1eb689c747507b30a30b8bfa67"
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  pruneopts = "UT"
+  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
+
+[[projects]]
+  branch = "master"
+  digest = "1:51a479a09b7ed06b7be5a854e27fcc328718ae0e5ad159f9ddeef12d0326c2e7"
+  name = "golang.org/x/time"
+  packages = ["rate"]
+  pruneopts = "UT"
+  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
+
+[[projects]]
+  digest = "1:a48f97fb737d5d61cf13e81cfef040942d217d086766b823757d39d4f6a4c547"
+  name = "google.golang.org/appengine"
+  packages = [
+    ".",
+    "internal",
+    "internal/app_identity",
+    "internal/base",
+    "internal/datastore",
+    "internal/log",
+    "internal/modules",
+    "internal/remote_api",
+    "internal/urlfetch",
+    "urlfetch",
+  ]
+  pruneopts = "UT"
+  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:39c2113f3a89585666e6f973650cff186b2d06deb4aa202c88addb87b0a201db"
+  name = "gopkg.in/russross/blackfriday.v2"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
+  version = "v2.0.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/PuerkitoBio/goquery",
+    "github.com/coreos/bbolt",
+    "github.com/dgrijalva/jwt-go",
+    "github.com/didip/tollbooth",
+    "github.com/didip/tollbooth_chi",
+    "github.com/globalsign/mgo",
+    "github.com/globalsign/mgo/bson",
+    "github.com/go-chi/chi",
+    "github.com/go-chi/chi/middleware",
+    "github.com/go-chi/cors",
+    "github.com/go-chi/render",
+    "github.com/go-pkgz/auth",
+    "github.com/go-pkgz/auth/avatar",
+    "github.com/go-pkgz/auth/provider",
+    "github.com/go-pkgz/auth/token",
+    "github.com/go-pkgz/lgr",
+    "github.com/go-pkgz/mongo",
+    "github.com/go-pkgz/repeater",
+    "github.com/go-pkgz/rest",
+    "github.com/go-pkgz/rest/cache",
+    "github.com/go-pkgz/rest/logger",
+    "github.com/google/uuid",
+    "github.com/gorilla/feeds",
+    "github.com/hashicorp/go-multierror",
+    "github.com/jessevdk/go-flags",
+    "github.com/microcosm-cc/bluemonday",
+    "github.com/patrickmn/go-cache",
+    "github.com/pkg/errors",
+    "github.com/rakyll/statik/fs",
+    "github.com/stretchr/testify/assert",
+    "github.com/stretchr/testify/require",
+    "golang.org/x/crypto/acme/autocert",
+    "golang.org/x/net/html",
+    "gopkg.in/russross/blackfriday.v2",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

backend/Gopkg.toml

@@ -12,10 +12,6 @@ required = ["github.com/patrickmn/go-cache"]
   name = "github.com/google/uuid"
   version = "0.2.0"
 
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-
 [[constraint]]
   branch = "master"
   name = "github.com/microcosm-cc/bluemonday"
@@ -24,13 +20,25 @@ required = ["github.com/patrickmn/go-cache"]
   name = "gopkg.in/russross/blackfriday.v2"
   version = "2.0.0"
 
+[[constraint]]
+  name = "github.com/patrickmn/go-cache"
+  version = "2.1.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/jessevdk/go-flags"
+
+[[constraint]]
+  name = "github.com/globalsign/mgo"
+  version = "r2018.06.15"
+
 [prune]
   go-tests = true
   unused-packages = true
 
-[[constraint]]
+
-  name = "github.com/patrickmn/go-cache"
+
-  version = "2.1.0"
+
 [[constraint]]
   branch = "master"
-  name = "github.com/jessevdk/go-flags"
+  name = "golang.org/x/crypto"

backend/app/cmd/avatar.go

@@ -0,0 +1,101 @@
+package cmd
+
+import (
+	"path"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/mongo"
+)
+
+// AvatarCommand set of flags and command for avatar migration
+// it converts all avatars from src.type to dst.type.
+// Note: it is possible to run migration for the same types (src = dst) in order to resize all avatars.
+type AvatarCommand struct {
+	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
+	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
+	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
+
+	migrator AvatarMigrator
+	CommonOpts
+}
+
+// AvatarMigrator defines interface for migration
+type AvatarMigrator interface {
+	Migrate(avatar.Store, avatar.Store) (int, error)
+}
+
+type avatarMigrator struct{}
+
+func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
+	return avatar.Migrate(dst, src)
+}
+
+// Execute runs  with AvatarCommand parameters, entry point for "avatar" command
+func (ac *AvatarCommand) Execute(args []string) error {
+	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
+
+	src, err := ac.makeAvatarStore(ac.AvatarSrc)
+	if err != nil {
+		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
+	}
+
+	dst, err := ac.makeAvatarStore(ac.AvatarDst)
+	if err != nil {
+		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
+	}
+
+	if ac.migrator == nil {
+		ac.migrator = avatarMigrator{}
+	}
+
+	count, err := ac.migrator.Migrate(dst, src)
+	if err != nil {
+		return err
+	}
+
+	if err = dst.Close(); err != nil {
+		log.Printf("[WARN] failed to close dst store %s", ac.AvatarDst.Type)
+	}
+	if err = src.Close(); err != nil {
+		log.Printf("[WARN] failed to close src store %s", ac.AvatarSrc.Type)
+	}
+
+	log.Printf("[INFO] completed, migrated avatars = %d", count)
+	return nil
+}
+
+func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
+	log.Printf("[DEBUG] make avatar store, type=%s", gr.Type)
+	switch gr.Type {
+	case "fs":
+		if err := makeDirs(gr.FS.Path); err != nil {
+			return nil, err
+		}
+		return avatar.NewLocalFS(gr.FS.Path), nil
+	case "mongo":
+		mgServer, err := ac.makeMongo()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create mongo server")
+		}
+		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
+		return avatar.NewGridFS(conn), nil
+	case "bolt":
+		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
+			return nil, err
+		}
+		return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
+	}
+	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
+}
+
+func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
+	if ac.Mongo.URL == "" {
+		return nil, errors.New("no mongo URL provided")
+	}
+	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
+}

backend/app/cmd/avatar_test.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/go-pkgz/auth/avatar"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAvatar_Execute(t *testing.T) {
+
+	mongoURL := os.Getenv("MONGO_TEST")
+	if mongoURL == "" {
+		mongoURL = "mongodb://localhost:27017/test"
+	}
+	if mongoURL == "skip" {
+		t.Skip("skip mongo app test")
+	}
+	defer os.RemoveAll("/tmp/ava-test")
+
+	// from fs to mongo
+	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
+	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+
+	// from fs to bolt
+	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
+	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test.db"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+
+	// failed
+	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
+	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
+		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.Error(t, err, "failed blah")
+}
+
+type avatarMigratorMock struct {
+	called   int
+	retError error
+	retCount int
+}
+
+func (a *avatarMigratorMock) Migrate(dst, src avatar.Store) (int, error) {
+	a.called++
+	return a.retCount, a.retError
+}

backend/app/cmd/backup.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// BackupCommand set of flags and command for export
+// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
+type BackupCommand struct {
+	ExportPath  string        `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
+	ExportFile  string        `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	CommonOpts
+}
+
+// Execute runs export with ExportCommand parameters, entry point for "export" command
+func (ec *BackupCommand) Execute(args []string) error {
+	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile}
+	fname, err := fp.parse(time.Now())
+	if err != nil {
+		return err
+	}
+
+	log.Printf("[DEBUG] export file %s", fname)
+
+	// prepare http client and request
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
+	defer cancel()
+	exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s", ec.RemarkURL, ec.Site)
+	req, err := http.NewRequest(http.MethodGet, exportURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "can't make export request for %s", exportURL)
+	}
+	req.SetBasicAuth("admin", ec.AdminPasswd)
+
+	// get with timeout
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", exportURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	fh, err := os.Create(fname)
+	if err != nil {
+		return errors.Wrapf(err, "can't create backup file %s", fname)
+	}
+	defer func() {
+		if err = fh.Close(); err != nil {
+			log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
+		}
+	}()
+
+	if _, err = io.Copy(fh, resp.Body); err != nil {
+		return errors.Wrapf(err, "failed to write backup file %s", fname)
+	}
+
+	log.Printf("[INFO] export completed, file %s", fname)
+	return nil
+}

backend/app/cmd/backup_test.go

@@ -0,0 +1,74 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBackup_Execute(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		fmt.Fprint(w, "blah\nblah2\n12345678\n")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	defer os.Remove("/tmp/remark-test.export")
+
+	data, err := ioutil.ReadFile("/tmp/remark-test.export")
+	require.Nil(t, err)
+	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
+}
+
+func TestBackup_ExecuteFailedStatus(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		w.WriteHeader(400)
+		fmt.Fprint(w, "some error")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
+}
+
+func TestBackup_ExecuteFailedWrite(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		fmt.Fprint(w, "blah\nblah2\n12345678\n")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
+		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
+}

backend/app/cmd/cleanup.go

@@ -0,0 +1,290 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// CleanupCommand set of flags and command for cleanup
+type CleanupCommand struct {
+	Site        string   `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Dry         bool     `long:"dry" description:"dry mode, will not remove comments"`
+	From        string   `long:"from" description:"from yyyymmdd"`
+	To          string   `long:"to" description:"from yyyymmdd"`
+	BadWords    []string `short:"w" long:"bword" description:"bad word(s)"`
+	BadUsers    []string `short:"u" long:"buser" description:"bad user(s)"`
+	AdminPasswd string   `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	SetTitle    bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+	CommonOpts
+}
+
+var (
+	defaultFrom = time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local)
+	defaultTo   = time.Date(2999, 1, 1, 0, 0, 0, 0, time.Local)
+)
+
+// Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
+// This command uses provided flags to detect and remove junk comments
+func (cc *CleanupCommand) Execute(args []string) error {
+	log.Printf("[INFO] cleanup for site %s", cc.Site)
+
+	posts, err := cc.postsInRange(cc.From, cc.To)
+	if err != nil {
+		return errors.Wrap(err, "can't get posts")
+	}
+	log.Printf("[DEBUG] got %d posts", len(posts))
+
+	totalComments, spamComments := 0, 0
+	for _, post := range posts {
+		comments, e := cc.listComments(post.URL)
+		if e != nil {
+			continue
+		}
+		totalComments += len(comments)
+
+		if cc.SetTitle {
+			cc.procTitles(comments)
+		} else {
+			spamComments += cc.procSpam(comments)
+
+		}
+	}
+
+	msg := fmt.Sprintf("comments=%d, spam=%d", totalComments, spamComments)
+	if cc.SetTitle {
+		msg = fmt.Sprintf("comments=%d", totalComments)
+	}
+
+	log.Printf("[INFO] completed, %s", msg)
+	return err
+}
+
+func (cc *CleanupCommand) procSpam(comments []store.Comment) int {
+	spamComments := 0
+	for _, comment := range comments {
+		spam, score := cc.isSpam(comment)
+		if spam {
+			spamComments++
+			if !cc.Dry {
+				if err := cc.deleteComment(comment); err != nil {
+					log.Printf("[WARN] can't remove comment, %v", err)
+				}
+			}
+			comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
+			log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+		}
+	}
+	return spamComments
+}
+
+func (cc *CleanupCommand) procTitles(comments []store.Comment) {
+	for _, comment := range comments {
+		if !cc.Dry {
+			if err := cc.setTitle(comment); err != nil {
+				log.Printf("[WARN] can't set title for comment, %v", err)
+			}
+		}
+	}
+}
+
+// get list of posts in from/to represented as yyyymmdd. this is [from-to] inclusive
+func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, error) {
+	posts, err := cc.listPosts()
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't list posts for %s", cc.Site)
+	}
+
+	from, to := defaultFrom, defaultTo
+
+	if fromS != "" {
+		from, err = time.ParseInLocation("20060102", fromS, time.Local)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't parse --from")
+		}
+	}
+
+	if toS != "" {
+		to, err = time.ParseInLocation("20060102", toS, time.Local)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't parse --to")
+		}
+	}
+
+	var filteredList []store.PostInfo
+	for _, postInfo := range posts {
+		if postInfo.FirstTS.After(from) && postInfo.LastTS.Before(to.AddDate(0, 0, 1)) {
+			filteredList = append(filteredList, postInfo)
+		}
+	}
+	return filteredList, nil
+}
+
+// get all posts via GET /list?site=siteID&limit=50&skip=10
+func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
+	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
+	r, err := http.Get(listURL)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
+	}
+	defer func() { _ = r.Body.Close() }()
+
+	if r.StatusCode != 200 {
+		return nil, errors.Errorf("request %s failed with status %d", listURL, r.StatusCode)
+	}
+
+	list := []store.PostInfo{}
+	if err = json.NewDecoder(r.Body).Decode(&list); err != nil {
+		return nil, errors.Wrapf(err, "can't decode list of posts for site %s", cc.Site)
+	}
+	return list, nil
+}
+
+// get all comments for post url via /find?site=siteID&url=post-url&format=[tree|plain]
+func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error) {
+
+	commentsURL := fmt.Sprintf("%s/api/v1/find?site=%s&url=%s&format=plain", cc.RemarkURL, cc.Site, postURL)
+
+	var r *http.Response
+	var err error
+
+	// handle 429 error from limiter
+	for {
+		r, err = http.Get(commentsURL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			_ = r.Body.Close()
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		break
+	}
+
+	defer func() { _ = r.Body.Close() }()
+
+	if r.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
+	}
+
+	commentsWithInfo := struct {
+		Comments []store.Comment `json:"comments"`
+		Info     store.PostInfo  `json:"info,omitempty"`
+	}{}
+
+	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
+	}
+	return commentsWithInfo.Comments, nil
+}
+
+// deleteComment with DELETE /admin/comment/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) deleteComment(c store.Comment) error {
+
+	deleteURL := fmt.Sprintf("%s/api/v1/admin/comment/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+	req, err := http.NewRequest("DELETE", deleteURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "failed to make delete request for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	req.SetBasicAuth("admin", cc.AdminPasswd)
+
+	client := http.Client{}
+	r, err := client.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "delete request failed for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	defer func() { _ = r.Body.Close() }()
+	if r.StatusCode != http.StatusOK {
+		return errors.Errorf("delete request failed with status %s", r.Status)
+	}
+	return nil
+}
+
+// setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) setTitle(c store.Comment) error {
+
+	titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+	req, err := http.NewRequest("PUT", titleURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	req.SetBasicAuth("admin", cc.AdminPasswd)
+
+	client := http.Client{}
+	r, err := client.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	defer func() { _ = r.Body.Close() }()
+	if r.StatusCode != http.StatusOK {
+		return errors.Errorf("title request failed with status %s", r.Status)
+	}
+	return nil
+}
+
+// isSpam calculates spam's probability as a score
+func (cc *CleanupCommand) isSpam(comment store.Comment) (bool, float64) {
+
+	badWord := func(txt string) float64 {
+		res := 0.0
+		for _, w := range cc.BadWords {
+			if strings.Contains(txt, w) {
+				res += 0.25
+			}
+			if res > 1 {
+				return 1
+			}
+		}
+		return res
+	}
+
+	hasBadUser := func(txt string) bool {
+		for _, w := range cc.BadUsers {
+			if strings.Contains(txt, w) {
+				return true
+			}
+		}
+		return false
+	}
+
+	score := 0.0
+
+	// don't mark deleted as spam
+	if comment.Deleted {
+		return false, 0
+	}
+
+	score += 50 * badWord(comment.Text) // up to 50, 4 bad words will reach max
+
+	if hasBadUser(comment.User.ID) { // predefined list of bad user substrings
+		score += 10
+	}
+
+	if comment.Score == 0 { // most of spam comments with 0 score
+		score += 20
+	}
+
+	// any link inside
+	if strings.Contains(comment.Text, "http:") || strings.Contains(comment.Text, "https:") {
+		score += 10
+	}
+
+	// 5 or more links
+	if strings.Count(comment.Text, "href") >= 5 {
+		score += 10
+	}
+
+	score = math.Max(score, 0)
+	score = math.Min(score, 100)
+
+	return score > 50, score
+}

backend/app/cmd/cleanup_test.go

@@ -0,0 +1,214 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+type cleanedComments struct {
+	ids  []string
+	lock sync.Mutex
+}
+
+func TestCleanup_IsSpam(t *testing.T) {
+	cc := CleanupCommand{
+		BadWords: []string{"bad1", "bad2", "very bad", "xyz"},
+		BadUsers: []string{"bu_"},
+	}
+
+	tbl := []struct {
+		text      string
+		user      string
+		score     int
+		isSpam    bool
+		spamScore float64
+		name      string
+	}{
+		{"", "", 1, false, 0, "empty passes"},
+		{"one very bad two blah bad1 bad2 http://xyz.com", "bu_user", 0, true, 90, "3badwords link 0score baduser"},
+		{"one very bad two blah bad1 bad2", "bu_user", 0, true, 67.5, "3 bad words 1score baduser"},
+		{"bad1 bad2 xyz very bad", "bu_user", 0, true, 80, "4badwords 0score baduser"},
+		{"bad1 bad2 xyz very bad", "user", 0, true, 70, "4badwords 0score"},
+		{"bad1 bad2 xyz very bad", "user", 1, false, 50, "4badwords 1score"},
+		{"bad1 test 12345", "user", 0, false, 32.5, "1badwords 0score"},
+	}
+
+	for n, tt := range tbl {
+		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
+		t.Run(checkName, func(t *testing.T) {
+			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
+			c.User.ID = tt.user
+			r, score := cc.isSpam(c)
+			assert.Equal(t, tt.isSpam, r)
+			assert.InDelta(t, tt.spamScore, score, 0.01)
+		})
+	}
+}
+
+func TestCleanup_postsInRange(t *testing.T) {
+
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	posts, err := cmd.postsInRange("20181218", "20181219")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(posts))
+
+	posts, err = cmd.postsInRange("", "")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(posts))
+
+	_, err = cmd.postsInRange("xxx", "yyy")
+	assert.NotNil(t, err)
+}
+
+func TestCleanup_listComments(t *testing.T) {
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
+	require.Nil(t, err)
+
+	comments, err := cmd.listComments("http://test.com/post1")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post2")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post-bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+}
+
+func TestCleanup_ExecuteSpam(t *testing.T) {
+	cleaned := cleanedComments{}
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, &cleaned)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
+		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("deleted %+v", cleaned.ids)
+	assert.Equal(t, []string{"/api/v1/admin/comment/1", "/api/v1/admin/comment/3", "/api/v1/admin/comment/11"}, cleaned.ids)
+}
+
+func TestCleanup_ExecuteTitle(t *testing.T) {
+	titledComments := cleanedComments{}
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, &titledComments)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("set titles for %+v", titledComments.ids)
+	assert.Equal(t, []string{"/api/v1/admin/title/1", "/api/v1/admin/title/2", "/api/v1/admin/title/3", "/api/v1/admin/title/11"}, titledComments.ids)
+}
+
+func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
+	r.HandleFunc("/api/v1/list", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
+		list := []store.PostInfo{
+			{
+				URL:     "http://test.com/post1",
+				FirstTS: time.Date(2018, 12, 17, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 17, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post2",
+				FirstTS: time.Date(2018, 12, 18, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 18, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post3",
+				FirstTS: time.Date(2018, 12, 19, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 19, 10, 30, 0, 0, time.Local),
+			},
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(list))
+	}))
+
+	r.HandleFunc("/api/v1/find", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "remark", r.URL.Query().Get("site"))
+		require.Equal(t, "plain", r.URL.Query().Get("format"))
+
+		commentsWithInfo := struct {
+			Comments []store.Comment `json:"comments"`
+			Info     store.PostInfo  `json:"info,omitempty"`
+		}{}
+
+		switch r.URL.Query().Get("url") {
+		case "http://test.com/post1":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "1", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+				{ID: "2", Text: "good one http://xyz.com", Score: 1, User: store.User{ID: "bu_user"}},
+				{ID: "3", Text: "http://xyz.com bad1 bad2", Score: 0, User: store.User{ID: "user"}},
+			}
+		case "http://test.com/post2":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "11", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+			}
+		case "http://test.com/post3":
+			commentsWithInfo.Comments = []store.Comment{}
+		}
+
+		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
+	}))
+
+	r.HandleFunc("/api/v1/admin/comment/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "DELETE", r.Method)
+		t.Log("delete ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	}))
+
+	r.HandleFunc("/api/v1/admin/title/{id}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PUT", r.Method)
+		t.Log("title for ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	}))
+
+}

backend/app/cmd/cmd.go

@@ -0,0 +1,110 @@
+// Package cmd has all top-level commands dispatched by main's flag.Parse
+// The entry point of each command is Execute function
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// CommonOptionsCommander extends flags.Commander with SetCommon
+// All commands should implement this interfaces
+type CommonOptionsCommander interface {
+	SetCommon(commonOpts CommonOpts)
+	Execute(args []string) error
+}
+
+// CommonOpts sets externally from main, shared across all commands
+type CommonOpts struct {
+	RemarkURL    string
+	SharedSecret string
+	Revision     string
+}
+
+// SetCommon satisfies CommonOptionsCommander interface and sets common option fields
+// The method called by main for each command
+func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
+	c.RemarkURL = commonOpts.RemarkURL
+	c.SharedSecret = commonOpts.SharedSecret
+	c.Revision = commonOpts.Revision
+}
+
+// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
+type fileParser struct {
+	site string
+	file string
+	path string
+}
+
+// parse apply template and also concat path and file. In case if file contains path separator path will be ignored
+func (p *fileParser) parse(now time.Time) (string, error) {
+
+	// file/location parameters my have template masks
+	fileTemplate := struct {
+		YYYYMMDD string
+		YYYY     string
+		YYYYMM   string
+		MM       string
+		DD       string
+		TS       string
+		UNIX     int64
+		SITE     string
+	}{
+		YYYYMMDD: now.Format("20060102"),
+		YYYY:     now.Format("2006"),
+		YYYYMM:   now.Format("200601"),
+		MM:       now.Format("01"),
+		DD:       now.Format("02"),
+		UNIX:     now.Unix(),
+		SITE:     p.site,
+		TS:       now.Format("20060102T150405"),
+	}
+
+	bb := bytes.Buffer{}
+	fname := p.file
+	if !strings.Contains(p.file, string(filepath.Separator)) {
+		fname = filepath.Join(p.path, p.file)
+	}
+
+	if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
+		return "", errors.Wrapf(err, "failed to parse %q", fname)
+	}
+	return bb.String(), nil
+}
+
+// resetEnv clears sensitive env vars
+func resetEnv(envs ...string) {
+	for _, env := range envs {
+		if err := os.Unsetenv(env); err != nil {
+			log.Printf("[WARN] can't unset env %s, %s", env, err)
+		}
+	}
+}
+
+// responseError returns error with status and response body
+func responseError(resp *http.Response) error {
+	body, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		body = []byte("")
+	}
+	return errors.Errorf("error response %q, %s", resp.Status, body)
+}
+
+// mkdir -p for all dirs
+func makeDirs(dirs ...string) error {
+	for _, dir := range dirs {
+		if err := os.MkdirAll(dir, 0700); err != nil { // If path is already a directory, MkdirAll does nothing
+			return errors.Wrapf(err, "can't make directory %s", dir)
+		}
+	}
+	return nil
+}

backend/app/cmd/cmd_test.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExport_ParseFileName(t *testing.T) {
+	tbl := []struct {
+		p   fileParser
+		res string
+		err bool
+	}{
+		{fileParser{}, "", false},
+		{fileParser{path: "/tmp/blah", file: "fname.gz"}, "/tmp/blah/fname.gz", false},
+		{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYYMMDD}}.gz"},
+			"/tmp/blah/fname-remark-20180821.gz", false},
+		{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
+			"/tmp/blah/fname-remark-2018-08.gz", false},
+		{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.YYYY}}-{{.MM}}.gz"},
+			"/tmp/fname-remark-2018-08.gz", false},
+		{fileParser{site: "remark", path: "/tmp/blah", file: "/tmp/fname-{{.SITE}}-{{.TS}}.gz"},
+			"/tmp/fname-remark-20180821T212615.gz", false},
+		{fileParser{site: "remark", path: "/tmp/blah", file: "fname-{{.XXX}}-{{.YYYY}}-{{.MM}}.gz"},
+			"", true},
+	}
+
+	now := time.Date(2018, 8, 21, 21, 26, 15, 0, time.UTC)
+	for i, tt := range tbl {
+		r, err := tt.p.parse(now)
+		if tt.err {
+			assert.NotNil(t, err)
+			continue
+		}
+		assert.Equal(t, tt.res, r, "check #%d", i)
+	}
+}

backend/app/cmd/import.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// ImportCommand set of flags and command for import
+type ImportCommand struct {
+	InputFile   string        `short:"f" long:"file" description:"input file name" required:"true"`
+	Provider    string        `short:"p" long:"provider" default:"disqus" choice:"disqus" choice:"wordpress" description:"import format"`
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	CommonOpts
+}
+
+// Execute runs import with ImportCommand parameters, entry point for "import" command
+func (ic *ImportCommand) Execute(args []string) error {
+	log.Printf("[INFO] import %s (%s), site %s", ic.InputFile, ic.Provider, ic.Site)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	reader, err := ic.reader(ic.InputFile)
+	if err != nil {
+		return errors.Wrapf(err, "can't open import file %s", ic.InputFile)
+	}
+
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), ic.Timeout)
+	defer cancel()
+	importURL := fmt.Sprintf("%s/api/v1/admin/import?site=%s&provider=%s", ic.RemarkURL, ic.Site, ic.Provider)
+	req, err := http.NewRequest(http.MethodPost, importURL, reader)
+	if err != nil {
+		return errors.Wrapf(err, "can't make import request for %s", importURL)
+	}
+	req.SetBasicAuth("admin", ic.AdminPasswd)
+
+	resp, err := client.Do(req.WithContext(ctx)) // closes request's reader
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", importURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "can't get response from importer")
+	}
+
+	log.Printf("[INFO] completed, status=%d, %s", resp.StatusCode, string(body))
+	return nil
+}
+
+// reader returns reader for file. For .gz file wraps with gunzip
+func (ic *ImportCommand) reader(inp string) (reader io.Reader, err error) {
+	inpFile, err := os.Open(inp)
+	if err != nil {
+		return nil, errors.Wrapf(err, "import failed, can't open %s", inp)
+	}
+
+	reader = inpFile
+	if strings.HasSuffix(ic.InputFile, ".gz") {
+		if reader, err = gzip.NewReader(inpFile); err != nil {
+			return nil, errors.Wrap(err, "can't make gz reader")
+		}
+	}
+	return reader, nil
+}

backend/app/cmd/import_test.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImport_Execute(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		body, err := ioutil.ReadAll(r.Body)
+		assert.Nil(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+
+		fmt.Fprintln(w, "some response")
+		fmt.Fprintln(w, string(body))
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt.gz", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}
+
+func TestImport_ExecuteFailed(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		fmt.Fprintln(w, "some response")
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import-no.txt", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	t.Log(err)
+	assert.NotNil(t, err, "fail on no such file")
+	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
+
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "http://127.0.0.1:12345", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	t.Log(err)
+	assert.NotNil(t, err, "fail on connection refused")
+	assert.True(t, strings.Contains(err.Error(), "connection refused"))
+
+	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("%+v", r)
+		w.WriteHeader(400)
+		fmt.Fprintln(w, "some response with 400")
+	}))
+	defer ts2.Close()
+	cmd = ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts2.URL, SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	t.Log(err)
+	assert.NotNil(t, err)
+}
+
+func TestImport_ExecuteTimeout(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		body, err := ioutil.ReadAll(r.Body)
+		assert.Nil(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+		time.Sleep(500 * time.Millisecond)
+		fmt.Fprintln(w, "some response")
+		fmt.Fprintln(w, string(body))
+
+	}))
+	defer ts.Close()
+
+	cmd := ImportCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--file=testdata/import.txt", "--timeout=300ms", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NotNil(t, err)
+	assert.True(t, strings.Contains(err.Error(), "deadline exceeded"))
+}

backend/app/cmd/restore.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+)
+
+// RestoreCommand set of flags and command for restore from backup
+type RestoreCommand struct {
+	ImportPath string `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
+	ImportFile string `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.YYYYMMDD}}.gz" description:"file name" required:"true"`
+
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"import timeout"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	CommonOpts
+}
+
+// Execute runs import with RestoreCommand parameters, entry point for "restore" command
+// uses ImportCommand with constructed full file name
+func (rc *RestoreCommand) Execute(args []string) error {
+	log.Printf("[INFO] restore %s, site %s", rc.ImportFile, rc.Site)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	fp := fileParser{site: rc.Site, path: rc.ImportPath, file: rc.ImportFile}
+	fname, err := fp.parse(time.Now())
+	if err != nil {
+		return err
+	}
+	importer := ImportCommand{
+		InputFile:   fname,
+		Site:        rc.Site,
+		Provider:    "native",
+		Timeout:     rc.Timeout,
+		AdminPasswd: rc.AdminPasswd,
+		CommonOpts:  rc.CommonOpts,
+	}
+	return importer.Execute(args)
+}

backend/app/cmd/restore_test.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	flags "github.com/jessevdk/go-flags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRestore_Execute(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/import")
+		assert.Equal(t, "POST", r.Method)
+		assert.Equal(t, "native", r.URL.Query().Get("provider"))
+		body, err := ioutil.ReadAll(r.Body)
+		assert.Nil(t, err)
+		assert.Equal(t, "blah\nblah2\n12345678\n", string(body))
+
+		fmt.Fprintln(w, "some response")
+		fmt.Fprintln(w, string(body))
+	}))
+	defer ts.Close()
+
+	cmd := RestoreCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=testdata", "--file=import.txt", "--admin-passwd=secret"})
+	require.Nil(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+}

backend/app/cmd/server.go

@@ -0,0 +1,579 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	auth_cache "github.com/patrickmn/go-cache"
+	"github.com/pkg/errors"
+
+	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/provider"
+	"github.com/go-pkgz/auth/token"
+	"github.com/go-pkgz/mongo"
+	"github.com/go-pkgz/rest/cache"
+
+	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/notify"
+	"github.com/umputun/remark/backend/app/rest/api"
+	"github.com/umputun/remark/backend/app/rest/proxy"
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+// ServerCommand with command line flags and env
+type ServerCommand struct {
+	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
+	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
+	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
+	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
+	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
+	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
+
+	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AdminPasswd     string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation  string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles  int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	ImageProxy      bool          `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
+	MaxCommentSize  int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes        int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	LowScore        int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore   int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	ReadOnlyAge     int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration    time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	Port            int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	WebRoot         string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit     float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords []string      `long:"restricted-words" env:"RESTRICTED_WORDS" default:"" description:"words prohibited to use in comments" env-delim:","`
+
+	Auth struct {
+		TTL struct {
+			JWT    time.Duration `long:"jwt" env:"JWT" default:"5m" description:"jwt TTL"`
+			Cookie time.Duration `long:"cookie" env:"COOKIE" default:"200h" description:"auth cookie TTL"`
+		} `group:"ttl" namespace:"ttl" env-namespace:"TTL"`
+		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
+		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
+		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
+		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
+		Dev      bool      `long:"dev" env:"DEV" description:"enable dev (local) oauth2"`
+	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
+
+	CommonOpts
+}
+
+// AuthGroup defines options group for auth params
+type AuthGroup struct {
+	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
+	CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
+}
+
+// StoreGroup defines options group for store params
+type StoreGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Bolt struct {
+		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
+		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+}
+
+// AvatarGroup defines options group for avatar params
+type AvatarGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/avatars.db" description:"avatars bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+}
+
+// CacheGroup defines options group for cache params
+type CacheGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Max  struct {
+		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
+		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
+		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
+	} `group:"max" namespace:"max" env-namespace:"MAX"`
+}
+
+// MongoGroup holds all mongo params, used by store, avatar and cache
+type MongoGroup struct {
+	URL string `long:"url" env:"URL" description:"mongo url"`
+	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
+}
+
+// AdminGroup defines options group for admin params
+type AdminGroup struct {
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Shared struct {
+		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
+		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
+	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+}
+
+// NotifyGroup defines options for notification
+type NotifyGroup struct {
+	Type      string `long:"type" env:"TYPE" description:"type of notification" choice:"none" choice:"telegram" default:"none"`
+	QueueSize int    `long:"queue" env:"QUEUE" description:"size of notification queue" default:"100"`
+	Telegram  struct {
+		Token   string        `long:"token" env:"TOKEN" description:"telegram token"`
+		Channel string        `long:"chan" env:"CHAN" description:"telegram channel"`
+		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"telegram timeout"`
+		API     string        `long:"api" env:"API" default:"https://api.telegram.org/bot" description:"telegram api prefix"`
+	} `group:"telegram" namespace:"telegram" env-namespace:"TELEGRAM"`
+}
+
+// SSLGroup defines options group for server ssl params
+type SSLGroup struct {
+	Type         string `long:"type" env:"TYPE" description:"ssl (auto)support" choice:"none" choice:"static" choice:"auto" default:"none"`
+	Port         int    `long:"port" env:"PORT" description:"port number for https server" default:"8443"`
+	Cert         string `long:"cert" env:"CERT" description:"path to cert.pem file"`
+	Key          string `long:"key" env:"KEY" description:"path to key.pem file"`
+	ACMELocation string `long:"acme-location" env:"ACME_LOCATION" description:"dir where certificates will be stored by autocert manager" default:"./var/acme"`
+	ACMEEmail    string `long:"acme-email" env:"ACME_EMAIL" description:"admin email for certificate notifications"`
+}
+
+// serverApp holds all active objects
+type serverApp struct {
+	*ServerCommand
+	restSrv       *api.Rest
+	migratorSrv   *api.Migrator
+	exporter      migrator.Exporter
+	devAuth       *provider.DevAuthServer
+	dataService   *service.DataStore
+	avatarStore   avatar.Store
+	notifyService *notify.Service
+	terminated    chan struct{}
+}
+
+// Execute is the entry point for "server" command, called by flag parser
+func (s *ServerCommand) Execute(args []string) error {
+	log.Printf("[INFO] start server on port %d", s.Port)
+	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC", "ADMIN_PASSWD")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { // catch signal and invoke graceful termination
+		stop := make(chan os.Signal, 1)
+		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
+		<-stop
+		log.Printf("[WARN] interrupt signal")
+		cancel()
+	}()
+
+	app, err := s.newServerApp()
+	if err != nil {
+		log.Printf("[PANIC] failed to setup application, %+v", err)
+	}
+	if err = app.run(ctx); err != nil {
+		log.Printf("[ERROR] remark terminated with error %+v", err)
+		return err
+	}
+	log.Printf("[INFO] remark terminated")
+	return nil
+}
+
+// newServerApp prepares application and return it with all active parts
+// doesn't start anything
+func (s *ServerCommand) newServerApp() (*serverApp, error) {
+
+	if err := makeDirs(s.BackupLocation); err != nil {
+		return nil, err
+	}
+
+	if !strings.HasPrefix(s.RemarkURL, "http://") && !strings.HasPrefix(s.RemarkURL, "https://") {
+		return nil, errors.Errorf("invalid remark42 url %s", s.RemarkURL)
+	}
+	log.Printf("[INFO] root url=%s", s.RemarkURL)
+
+	storeEngine, err := s.makeDataStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make data store engine")
+	}
+
+	adminStore, err := s.makeAdminStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make admin store")
+	}
+
+	dataService := &service.DataStore{
+		Interface:              storeEngine,
+		EditDuration:           s.EditDuration,
+		AdminStore:             adminStore,
+		MaxCommentSize:         s.MaxCommentSize,
+		MaxVotes:               s.MaxVotes,
+		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
+		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
+	}
+
+	loadingCache, err := s.makeCache()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make cache")
+	}
+
+	avatarStore, err := s.makeAvatarStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make avatar store")
+	}
+	authenticator := s.makeAuthenticator(dataService, avatarStore, adminStore)
+
+	exporter := &migrator.Native{DataStore: dataService}
+
+	migr := &api.Migrator{
+		Cache:             loadingCache,
+		NativeImporter:    &migrator.Native{DataStore: dataService},
+		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
+		WordPressImporter: &migrator.WordPress{DataStore: dataService},
+		NativeExporter:    &migrator.Native{DataStore: dataService},
+		KeyStore:          adminStore,
+	}
+
+	notifyService, err := s.makeNotify(dataService)
+	if err != nil {
+		log.Printf("[WARN] failed to make notify service, %s", err)
+		notifyService = notify.NopService // disable notifier
+	}
+
+	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
+	commentFormatter := store.NewCommentFormatter(imgProxy)
+
+	sslConfig, err := s.makeSSLConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make config of ssl server params")
+	}
+
+	srv := &api.Rest{
+		Version:          s.Revision,
+		DataService:      dataService,
+		WebRoot:          s.WebRoot,
+		RemarkURL:        s.RemarkURL,
+		ImageProxy:       imgProxy,
+		CommentFormatter: commentFormatter,
+		Migrator:         migr,
+		ReadOnlyAge:      s.ReadOnlyAge,
+		SharedSecret:     s.SharedSecret,
+		Authenticator:    authenticator,
+		Cache:            loadingCache,
+		NotifyService:    notifyService,
+		SSLConfig:        sslConfig,
+		UpdateLimiter:    s.UpdateLimit,
+	}
+
+	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
+
+	var devAuth *provider.DevAuthServer
+	if s.Auth.Dev {
+		da, err := authenticator.DevAuth()
+		if err != nil {
+			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+		}
+		devAuth = da
+	}
+
+	return &serverApp{
+		ServerCommand: s,
+		restSrv:       srv,
+		migratorSrv:   migr,
+		exporter:      exporter,
+		devAuth:       devAuth,
+		dataService:   dataService,
+		avatarStore:   avatarStore,
+		notifyService: notifyService,
+		terminated:    make(chan struct{}),
+	}, nil
+}
+
+// Run all application objects
+func (a *serverApp) run(ctx context.Context) error {
+	if a.AdminPasswd != "" {
+		log.Printf("[WARN] admin basic auth enabled")
+	}
+
+	go func() {
+		// shutdown on context cancellation
+		<-ctx.Done()
+		log.Print("[INFO] shutdown initiated")
+		a.restSrv.Shutdown()
+		if a.devAuth != nil {
+			a.devAuth.Shutdown()
+		}
+		if e := a.dataService.Close(); e != nil {
+			log.Printf("[WARN] failed to close data store, %s", e)
+		}
+		if e := a.avatarStore.Close(); e != nil {
+			log.Printf("[WARN] failed to close avatar store, %s", e)
+		}
+		a.notifyService.Close()
+		log.Print("[INFO] shutdown completed")
+	}()
+	a.activateBackup(ctx) // runs in goroutine for each site
+	if a.Auth.Dev {
+		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
+	}
+	a.restSrv.Run(a.Port)
+	close(a.terminated)
+	return nil
+}
+
+// Wait for application completion (termination)
+func (a *serverApp) Wait() {
+	<-a.terminated
+}
+
+// activateBackup runs background backups for each site
+func (a *serverApp) activateBackup(ctx context.Context) {
+	for _, siteID := range a.Sites {
+		backup := migrator.AutoBackup{
+			Exporter:       a.exporter,
+			BackupLocation: a.BackupLocation,
+			SiteID:         siteID,
+			KeepMax:        a.MaxBackupFiles,
+			Duration:       24 * time.Hour,
+		}
+		go backup.Do(ctx)
+	}
+}
+
+// makeDataStore creates store for all sites
+func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
+	log.Printf("[INFO] make data store, type=%s", s.Store.Type)
+
+	switch s.Store.Type {
+	case "bolt":
+		if err = makeDirs(s.Store.Bolt.Path); err != nil {
+			return nil, errors.Wrap(err, "failed to create bolt store")
+		}
+		sites := []engine.BoltSite{}
+		for _, site := range s.Sites {
+			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
+		}
+		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
+	case "mongo":
+		mgServer, e := s.makeMongo()
+		if e != nil {
+			return result, errors.Wrap(e, "failed to create mongo server")
+		}
+		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
+		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+	default:
+		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
+	}
+	return result, errors.Wrap(err, "can't initialize data store")
+}
+
+func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
+	log.Printf("[INFO] make avatar store, type=%s", s.Avatar.Type)
+
+	switch s.Avatar.Type {
+	case "fs":
+		if err := makeDirs(s.Avatar.FS.Path); err != nil {
+			return nil, err
+		}
+		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
+	case "mongo":
+		mgServer, err := s.makeMongo()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create mongo server")
+		}
+		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
+		return avatar.NewGridFS(conn), nil
+	case "bolt":
+		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
+			return nil, err
+		}
+		return avatar.NewBoltDB(s.Avatar.Bolt.File, bolt.Options{})
+	}
+	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
+}
+
+func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
+	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
+
+	switch s.Admin.Type {
+	case "shared":
+		if s.Admin.Shared.Email == "" { // no admin email, use admin@domain
+			if u, err := url.Parse(s.RemarkURL); err == nil {
+				s.Admin.Shared.Email = "admin@" + u.Host
+			}
+		}
+		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
+	case "mongo":
+		mgServer, e := s.makeMongo()
+		if e != nil {
+			return nil, errors.Wrap(e, "failed to create mongo server")
+		}
+		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
+		return admin.NewMongoStore(conn, s.SharedSecret), nil
+	default:
+		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
+	}
+}
+
+func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
+	log.Printf("[INFO] make cache, type=%s", s.Cache.Type)
+	switch s.Cache.Type {
+	case "mem":
+		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+			cache.MaxKeys(s.Cache.Max.Items))
+	// case "mongo":
+	// 	mgServer, err := s.makeMongo()
+	// 	if err != nil {
+	// 		return nil, errors.Wrap(err, "failed to create mongo server")
+	// 	}
+	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
+	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
+	// 		cache.MaxKeys(s.Cache.Max.Items))
+	case "none":
+		return &cache.Nop{}, nil
+	}
+	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
+}
+
+func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
+	if s.Mongo.URL == "" {
+		return nil, errors.New("no mongo URL provided")
+	}
+	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
+}
+
+func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
+
+	providers := 0
+	if s.Auth.Google.CID != "" && s.Auth.Google.CSEC != "" {
+		authenticator.AddProvider("google", s.Auth.Google.CID, s.Auth.Google.CSEC)
+		providers++
+	}
+	if s.Auth.Github.CID != "" && s.Auth.Github.CSEC != "" {
+		authenticator.AddProvider("github", s.Auth.Github.CID, s.Auth.Github.CSEC)
+		providers++
+	}
+	if s.Auth.Facebook.CID != "" && s.Auth.Facebook.CSEC != "" {
+		authenticator.AddProvider("facebook", s.Auth.Facebook.CID, s.Auth.Facebook.CSEC)
+		providers++
+	}
+	if s.Auth.Yandex.CID != "" && s.Auth.Yandex.CSEC != "" {
+		authenticator.AddProvider("yandex", s.Auth.Yandex.CID, s.Auth.Yandex.CSEC)
+		providers++
+	}
+	if s.Auth.Dev {
+		authenticator.AddProvider("dev", "", "")
+		providers++
+	}
+
+	if providers == 0 {
+		log.Printf("[WARN] no auth providers defined")
+	}
+}
+
+func (s *ServerCommand) makeNotify(dataStore *service.DataStore) (*notify.Service, error) {
+	log.Printf("[INFO] make notify, type=%s", s.Notify.Type)
+	switch s.Notify.Type {
+	case "telegram":
+		tg, err := notify.NewTelegram(s.Notify.Telegram.Token, s.Notify.Telegram.Channel,
+			s.Notify.Telegram.Timeout, s.Notify.Telegram.API)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create telegram notification destination")
+		}
+		return notify.NewService(dataStore, s.Notify.QueueSize, tg), nil
+	case "none":
+		return notify.NopService, nil
+	}
+	return nil, errors.Errorf("unsupported notification type %q", s.Notify.Type)
+}
+
+func (s *ServerCommand) makeSSLConfig() (config api.SSLConfig, err error) {
+	switch s.SSL.Type {
+	case "none":
+		config.SSLMode = api.None
+	case "static":
+		if s.SSL.Cert == "" {
+			return config, errors.New("path to cert.pem is required")
+		}
+		if s.SSL.Key == "" {
+			return config, errors.New("path to key.pem is required")
+		}
+		config.SSLMode = api.Static
+		config.Port = s.SSL.Port
+		config.Cert = s.SSL.Cert
+		config.Key = s.SSL.Key
+	case "auto":
+		config.SSLMode = api.Auto
+		config.Port = s.SSL.Port
+		config.ACMELocation = s.SSL.ACMELocation
+		if s.SSL.ACMEEmail != "" {
+			config.ACMEEmail = s.SSL.ACMEEmail
+		} else if s.Admin.Type == "shared" && s.Admin.Shared.Email != "" {
+			config.ACMEEmail = s.Admin.Shared.Email
+		} else if u, e := url.Parse(s.RemarkURL); e == nil {
+			config.ACMEEmail = "admin@" + u.Hostname()
+		}
+	}
+	return config, err
+}
+
+func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Store, admns admin.Store) *auth.Service {
+	authenticator := auth.NewService(auth.Opts{
+		URL:            strings.TrimSuffix(s.RemarkURL, "/"),
+		Issuer:         "remark42",
+		TokenDuration:  s.Auth.TTL.JWT,
+		CookieDuration: s.Auth.TTL.Cookie,
+		SecureCookies:  strings.HasPrefix(s.RemarkURL, "https://"),
+		SecretReader: token.SecretFunc(func() (string, error) { // get secret per site
+			return admns.Key()
+		}),
+		ClaimsUpd: token.ClaimsUpdFunc(func(c token.Claims) token.Claims { // set attributes, on new token or refresh
+			if c.User == nil {
+				return c
+			}
+			c.User.SetAdmin(ds.IsAdmin(c.Audience, c.User.ID))
+			c.User.SetBoolAttr("blocked", ds.IsBlocked(c.Audience, c.User.ID))
+			return c
+		}),
+		AdminPasswd: s.AdminPasswd,
+		Validator: token.ValidatorFunc(func(token string, claims token.Claims) bool { // check on each auth call (in middleware)
+			if claims.User == nil {
+				return false
+			}
+			return !claims.User.BoolAttr("blocked")
+		}),
+		AvatarStore:       avas,
+		AvatarResizeLimit: s.Avatar.RszLmt,
+		AvatarRoutePath:   "/api/v1/avatar",
+		Logger:            log.Default(),
+		RefreshCache:      newAuthRefreshCache(),
+	})
+	s.addAuthProviders(authenticator)
+	return authenticator
+}
+
+// authRefreshCache used by authenticator to minimize repeatable token refreshes
+type authRefreshCache struct {
+	*auth_cache.Cache
+}
+
+func newAuthRefreshCache() *authRefreshCache {
+	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+}
+
+func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
+	return c.Cache.Get(key.(string))
+}
+
+func (c *authRefreshCache) Set(key, value interface{}) {
+	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+}

backend/app/cmd/server_test.go

@@ -0,0 +1,419 @@
+package cmd
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/globalsign/mgo"
+	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/mongo"
+	flags "github.com/jessevdk/go-flags"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServerApp(t *testing.T) {
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18080
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	// send ping
+	resp, err := http.Get("http://localhost:18080/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	// add comment
+	client := http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	req.SetBasicAuth("admin", "password")
+	require.Nil(t, err)
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+	body, _ = ioutil.ReadAll(resp.Body)
+	t.Log(string(body))
+
+	assert.Equal(t, "admin@demo.remark42.com", app.dataService.AdminStore.Email(""), "default admin email")
+
+	app.Wait()
+}
+
+func TestServerApp_DevMode(t *testing.T) {
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18085
+		o.AdminPasswd = "password"
+		o.Auth.Dev = true
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
+	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
+	// send ping
+	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	app.Wait()
+}
+
+func TestServerApp_WithMongo(t *testing.T) {
+
+	mongoURL := os.Getenv("MONGO_TEST")
+	if mongoURL == "" {
+		mongoURL = "mongodb://localhost:27017/test"
+	}
+	if mongoURL == "skip" {
+		t.Skip("skip mongo app test")
+	}
+
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--cache.type=none", "--store.type=mongo",
+		"--avatar.type=mongo", "--mongo.url=" + mongoURL, "--mongo.db=test_remark", "--port=12345", "--admin.type=mongo"})
+	require.Nil(t, err)
+	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
+	opts.BackupLocation = "/tmp"
+
+	// create app
+	app, err := opts.newServerApp()
+	require.Nil(t, err)
+
+	defer func() {
+		s, err := mongo.NewServerWithURL(mongoURL, 10*time.Second)
+		assert.NoError(t, err)
+		conn := mongo.NewConnection(s, "test_remark", "")
+		_ = conn.WithDB(func(dbase *mgo.Database) error {
+			assert.NoError(t, dbase.DropDatabase())
+			return nil
+		})
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(5 * time.Second)
+		log.Print("[TEST] terminate app")
+		cancel()
+	}()
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	// send ping
+	resp, err := http.Get("http://localhost:12345/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	app.Wait()
+}
+
+func TestServerApp_WithSSL(t *testing.T) {
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://localhost:18443", SharedSecret: "123456"})
+
+	// prepare options
+	p := flags.NewParser(&opts, flags.Default)
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--port=18080", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt", "--avatar.bolt.file=/tmp/ava-test.db", "--notify.type=none",
+		"--ssl.type=static", "--ssl.cert=testdata/cert.pem", "--ssl.key=testdata/key.pem", "--ssl.port=18443"})
+	require.Nil(t, err)
+
+	// create app
+	app, err := opts.newServerApp()
+	require.Nil(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(1 * time.Second)
+		log.Print("[TEST] terminate app")
+		cancel()
+	}()
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	client := http.Client{
+		// prevent http redirect
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+
+		// allow self-signed certificate
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	// check http to https redirect response
+	resp, err := client.Get("http://localhost:18080/blah?param=1")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 307, resp.StatusCode)
+	assert.Equal(t, "https://localhost:18443/blah?param=1", resp.Header.Get("Location"))
+
+	// check https server
+	resp, err = client.Get("https://localhost:18443/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	app.Wait()
+}
+
+func TestServerApp_Failed(t *testing.T) {
+	opts := ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&opts, flags.Default)
+
+	// RO bolt location
+	_, err := p.ParseArgs([]string{"--backup=/tmp", "--store.bolt.path=/dev/null"})
+	assert.Nil(t, err)
+	_, err = opts.newServerApp()
+	assert.EqualError(t, err, "failed to make data store engine: failed to create bolt store: can't make directory /dev/null: mkdir /dev/null: not a directory")
+	t.Log(err)
+
+	// RO backup location
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	_, err = p.ParseArgs([]string{"--store.bolt.path=/tmp", "--backup=/dev/null/not-writable"})
+	assert.Nil(t, err)
+	_, err = opts.newServerApp()
+	assert.EqualError(t, err, "can't make directory /dev/null/not-writable: mkdir /dev/null: not a directory")
+	t.Log(err)
+
+	// invalid url
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "demo.remark42.com", SharedSecret: "123456"})
+
+	_, err = p.ParseArgs([]string{"--backup=/tmp", "----store.bolt.path=/tmp"})
+	assert.Nil(t, err)
+	_, err = opts.newServerApp()
+	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
+	t.Log(err)
+
+	opts = ServerCommand{}
+	opts.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	_, err = p.ParseArgs([]string{"--backup=/tmp", "--store.type=blah"})
+	assert.NotNil(t, err, "blah is invalid type")
+
+	opts.Store.Type = "blah"
+	_, err = opts.newServerApp()
+	assert.EqualError(t, err, "failed to make data store engine: unsupported store type blah")
+	t.Log(err)
+}
+
+func TestServerApp_Shutdown(t *testing.T) {
+	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18090
+		return o
+	})
+	st := time.Now()
+	err := app.run(ctx)
+	assert.Nil(t, err)
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+	app.Wait()
+}
+
+func TestServerApp_MainSignal(t *testing.T) {
+
+	go func() {
+		time.Sleep(250 * time.Millisecond)
+		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.Nil(t, err)
+	}()
+	st := time.Now()
+
+	s := ServerCommand{}
+	s.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "123456"})
+
+	p := flags.NewParser(&s, flags.Default)
+	args := []string{"test", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.type=bolt",
+		"--avatar.bolt.file=/tmp/ava-test.db", "--port=18100", "--notify.type=none"}
+	defer os.Remove("/tmp/ava-test.db")
+	_, err := p.ParseArgs(args)
+	require.Nil(t, err)
+	err = s.Execute(args)
+	assert.NoError(t, err, "execute failed")
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+}
+
+func Test_ACMEEmail(t *testing.T) {
+	cmd := ServerCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "https://remark.com:443", SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	args := []string{"--ssl.type=auto"}
+	_, err := p.ParseArgs(args)
+	require.Nil(t, err)
+	cfg, err := cmd.makeSSLConfig()
+	require.Nil(t, err)
+	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
+
+	cmd = ServerCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "https://remark.com", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	args = []string{"--ssl.type=auto", "--ssl.acme-email=adminname@adminhost.com"}
+	_, err = p.ParseArgs(args)
+	require.Nil(t, err)
+	cfg, err = cmd.makeSSLConfig()
+	require.Nil(t, err)
+	assert.Equal(t, "adminname@adminhost.com", cfg.ACMEEmail)
+
+	cmd = ServerCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "https://remark.com", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	args = []string{"--ssl.type=auto", "--admin.type=shared", "--admin.shared.email=superadmin@admin.com"}
+	_, err = p.ParseArgs(args)
+	require.Nil(t, err)
+	cfg, err = cmd.makeSSLConfig()
+	require.Nil(t, err)
+	assert.Equal(t, "superadmin@admin.com", cfg.ACMEEmail)
+
+	cmd = ServerCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "https://remark.com:443", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	args = []string{"--ssl.type=auto", "--admin.type=shared"}
+	_, err = p.ParseArgs(args)
+	require.Nil(t, err)
+	cfg, err = cmd.makeSSLConfig()
+	require.Nil(t, err)
+	assert.Equal(t, "admin@remark.com", cfg.ACMEEmail)
+}
+
+func TestServerAuthHooks(t *testing.T) {
+	app, ctx := prepServerApp(t, 10000*time.Millisecond, func(o ServerCommand) ServerCommand {
+		o.Port = 18080
+		return o
+	})
+
+	go func() { _ = app.run(ctx) }()
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	// make a token for user dev
+	tkService := app.restSrv.Authenticator.TokenService()
+	tkService.TokenDuration = time.Second
+
+	claims := token.Claims{
+		StandardClaims: jwt.StandardClaims{
+			Audience:  "remark",
+			Issuer:    "remark",
+			ExpiresAt: time.Now().Add(time.Second).Unix(),
+			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
+		},
+		User: &token.User{
+			ID:   "dev",
+			Name: "developer one",
+		},
+	}
+	tk, err := tkService.Token(claims)
+	require.NoError(t, err)
+	t.Log(tk)
+
+	// add comment
+	client := http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
+	req.Header.Set("X-JWT", tk)
+	require.Nil(t, err)
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
+
+	// block user dev as admin
+	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
+	assert.Nil(t, e)
+	req.SetBasicAuth("admin", "password")
+	resp, e = client.Do(req)
+	require.Nil(t, e)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "user dev blocked")
+	b, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	t.Log(string(b))
+
+	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
+
+	// try add a comment with blocked user
+	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
+	req.Header.Set("X-JWT", tk)
+	require.Nil(t, err)
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode, "blocked user can't post")
+
+	app.Wait()
+}
+
+func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand) ServerCommand) (*serverApp, context.Context) {
+	cmd := ServerCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: "https://demo.remark42.com", SharedSecret: "secret"})
+
+	// prepare options
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--admin-passwd=password", "--site=remark"})
+	require.Nil(t, err)
+	cmd.Avatar.FS.Path, cmd.Avatar.Type, cmd.BackupLocation = "/tmp", "fs", "/tmp"
+	cmd.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", cmd.Port)
+	cmd.Store.Bolt.Timeout = 10 * time.Second
+	cmd.Auth.Github.CSEC, cmd.Auth.Github.CID = "csec", "cid"
+	cmd.Auth.Google.CSEC, cmd.Auth.Google.CID = "csec", "cid"
+	cmd.Auth.Facebook.CSEC, cmd.Auth.Facebook.CID = "csec", "cid"
+	cmd.Auth.Yandex.CSEC, cmd.Auth.Yandex.CID = "csec", "cid"
+	cmd.BackupLocation = "/tmp"
+	cmd.Notify.Type = "telegram"
+	cmd.Notify.Telegram.API = "http://127.0.0.1:12340/"
+	cmd.Notify.Telegram.Token = "blah"
+	cmd = fn(cmd)
+
+	os.Remove(cmd.Store.Bolt.Path + "/remark.db")
+
+	// create app
+	app, err := cmd.newServerApp()
+	require.Nil(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(duration)
+		log.Print("[TEST] terminate app")
+		cancel()
+	}()
+	return app, ctx
+}

backend/app/cmd/testdata/cert.pem

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEpDCCAowCCQCDOGCCov0x5zANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMTgxMDIwMTIxNjI0WhcNMjgxMDE3MTIxNjI0WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDi
+PU80EAvPZizxJhwG1X2XBZv8iVgsTIAJhCYSzLScgn46Yln9s1/LJFtQloDrZByk
+m+/3VfaYsrwUT8Xn9NHYnde9q/L9sQM9rfLsVjHkLp3gPaWYRburMfOqrdxZ2GYR
+uUi9Ni4s7o4WE1CBCoepiFdVY3OYKQdifygCqZlsAgywkqz7q0qJH+Fkw4OTzf+o
+fqr6keF8B6HbHf4Kgwv0/h4EKqZ2YeHqn7bCVS/ppr6t5n7+s/EjN6gD5WnBsxNU
+ILjnREW4wG6rN0gY9IaZ6vDeJBqe9fhYPevUH9OZInOlYpHYHt1dHo3sxto64KAp
+eZv9/LUuO0QSBOzh6L20HLSqUMQVgsVxOjXpza0DTosQfpEKTkuJxMeedweWClRN
+gZY6p7xdFtKju43dfxLsUtSLq3TGFVHWVwtPuSdIDwFRx5JPkL5WgK8XEsN9k7Hs
+oA+/Ng98hvNj1kSQi8FXTFnlZ7+YNi51UbhveGCKRZdh6Klm1HwYxQcY6kOWRyv+
+QNyeIKlIxhfgsL2iJzZMkM/OL/IXwCROCha/v9tMk7gKmlS7O2MHNILcGP1LsyFJ
+EI67ep7LK4q8EZfVPThjS/cTag5zETWoaJ7cxY36WxaeQB+3OgKT0NRiAT88+eKq
+Yd5nRJ6LSdSM2rQwqiW8JF4T/Xq10Y0bGoKoqf/ywQIDAQABMA0GCSqGSIb3DQEB
+CwUAA4ICAQANcjV3Rzb7sefN7XR77tKpIr0TT2Qk/s4V6hV91vDq+G+7YGi+BRaX
+6VWd1OjQVq4YgSmOgQ7L11xEMIZXsNlerwIV0DqLYFYD1Nj6iYdvis4JWDSbFUlI
+gJNyvEhC5dtGcHg8mWYGWcQE5Eu2paVrL49madExqvlpIk0cfJVhdviI9t15mJ+X
+JBbKaxdoh7CV5oR5cbTJqkszksozMH6krqGZAOgOAn5KzqIZFNyeDUYzPhzjySlh
+NNO/sdMyRFfNqZh/bC84z7HB9/KxHnFTBUvUdoX0JRAnkSV5zTuqGFxP/MRdNC1Y
+1hre9nnMSzHCoyizFihKBtaTc5OKbV62rl20WSSrDuNzFBGK3jHA3qnL7pdVacXg
++TOt9g0YoKLxQTPzvRGLo3kZbELP2dWzK8ILZeSi2vdJJRGe4Vt3oYF3Ky79ZFW4
+wIm8QTpOKHQlH2mpw+OCQ42AKJdBT4f/stJDxuKk7fBHAEzEnH94pLGpi23BJNGR
+wSOk+h6Oi0CHEO+Iww5nWX+y4msK7mZwl3eBdYmuXQJnODyGj3n2rjB0Rj51BNXV
+APPpGaCbBEDdnTD7iNtwj7By/kVFmzfjAX45C1aF/3B2RS8AAEyEIP1uolykSQQJ
+Cdt0GVjDnKk4nG8//FsPXBU9s7UFYa2E68Oz2JUyOyHVKt0xkc0Seg==
+-----END CERTIFICATE-----

backend/app/cmd/testdata/import.txt

@@ -0,0 +1,3 @@
+blah
+blah2
+12345678

backend/app/cmd/testdata/key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDiPU80EAvPZizx
+JhwG1X2XBZv8iVgsTIAJhCYSzLScgn46Yln9s1/LJFtQloDrZBykm+/3VfaYsrwU
+T8Xn9NHYnde9q/L9sQM9rfLsVjHkLp3gPaWYRburMfOqrdxZ2GYRuUi9Ni4s7o4W
+E1CBCoepiFdVY3OYKQdifygCqZlsAgywkqz7q0qJH+Fkw4OTzf+ofqr6keF8B6Hb
+Hf4Kgwv0/h4EKqZ2YeHqn7bCVS/ppr6t5n7+s/EjN6gD5WnBsxNUILjnREW4wG6r
+N0gY9IaZ6vDeJBqe9fhYPevUH9OZInOlYpHYHt1dHo3sxto64KApeZv9/LUuO0QS
+BOzh6L20HLSqUMQVgsVxOjXpza0DTosQfpEKTkuJxMeedweWClRNgZY6p7xdFtKj
+u43dfxLsUtSLq3TGFVHWVwtPuSdIDwFRx5JPkL5WgK8XEsN9k7HsoA+/Ng98hvNj
+1kSQi8FXTFnlZ7+YNi51UbhveGCKRZdh6Klm1HwYxQcY6kOWRyv+QNyeIKlIxhfg
+sL2iJzZMkM/OL/IXwCROCha/v9tMk7gKmlS7O2MHNILcGP1LsyFJEI67ep7LK4q8
+EZfVPThjS/cTag5zETWoaJ7cxY36WxaeQB+3OgKT0NRiAT88+eKqYd5nRJ6LSdSM
+2rQwqiW8JF4T/Xq10Y0bGoKoqf/ywQIDAQABAoICAGZpwIfd16rMIv4K9Vb2n8KU
+rHcWKU7sVm14X8/U0NGklMWTLg+VeoC39Yo2hyeHixvM50T8qgOXrWI4dms5PSEk
+2rumsfm7fq8WJkhsAGu92oPfBC45KuolU5to1r3pItNVH6Nfemmml7VQivnPLXhG
+GtqxiWtQ72HcE92YrrXg+6OW0i2e7b/DKcH2a4KmFtxMctO44+zEe7V4Dz7s8aW2
+vxm9wNsIFj+iixXPgD/x1pFcLRLZnAgIMsuIt2YSzWYg5fLyyekHW1GwVbfOQXw9
+iUnV6CMbT4+KgE3nFl9U5CrywviliCmWIkMUBUfv1TDRNxczpUTHWVoa1qyxMeYT
+ubfGoBjKzf6/bvTEm/EkBDNZWOJ+fvQZAK8f8Pq6K7KeW61kfW/dM8YzKcKKelXc
+yeE+6WG7BH4rLmVRh1UyFfT59oipsTKYwKekDSLCDDhXkpIMV6DCvnPmdO1o/7ed
+13VNA2cCV+3Sj0BI3hob74jxo/UflkGX9SGRe7LpYfRkIyjII4ZjCPBiDNSOMKm+
+ygRjiPmwjZfmDPgd5EicmAhSb7TdzUmdMssz7z2OWHX97beEt2lV0fIft/xb4u90
+eGsafasnY4qfVyJolJ4FYQkkQlvZWoqaBB7cN8VhLG+eU1tqYz6baUOwY0NvWY3E
+iekbQ5u7RWVIQg7cuHtBAoIBAQD32jXlQjO9oNBLcibYUQlZWKkQIVcWpTCtMBbC
+ZqqV0+Xlk11SPYtWm2x1pUXp22wZk+AFCzskioWfGLBy4Br9GEF3iXLna4uXnb/6
+JdzP4i1kpUk3/F1GJCT8GTkps2bdGhYwhkaWK9IIlAN7zoq5t1GA94kYTrYepvCp
+SR2pbgpwvqfqe+LTW5xNtOVq+48p9oXuac8YTXBDci7hAzUjLkkC1UAWsummAC/o
+1BM7NUB2X3PorIofYG+4qgLHlmDiV0On470J8QZIv79ifMM+ER/a0UjHrmoFErbr
+Ze+bAjXSgkfQIihUwXdL6xEB0vinsmaNmGpmkBhSjrwforbJAoIBAQDprTdzyKVM
+T7e6I2V1AyaCokBhQfVfn+fTpewgNyw7iYBq6udXoNUPFI7TG23F01/R+0iShEmW
+u1hZeKdnX6r5SkvLOC/wZxPCAptfTHs+cKjOiwWXIdNA2dG6rKz50n8cuEDf1uLl
+/jN3JLmlyuYZStMVxVOsUoPPTBlASSgbFvaz8mTgol64hUUp335UTp80XteZHWAw
++IZMCp769E3u+87XkaSIZWRw0Grgbsr8XI3rNLq0ZPOtRuvnE1LWAkgSa2Syouru
+2a8WIaFUIUlAdfPYFij3qZm8lI5s1WEj8QZnGvDViCgeTASsb4mm/GKZNbqp+kZj
+YFgvuUDq+0A5AoIBAQCT9E0MCFL+tgrhnskVhHqCPWze1ig4um+uUvCjJ8pZGl84
+hpnEyDxvfORn7jsn+PY05pNqL2Qk0hQl/wUF7kXuJBaglwpin057OY4qc4O3DRtv
+40MsCphbkKQLYbs/63pSLopkeSOV7Z65xDG4LHmfl5Et3ZaXThVlgt5TD/SG+ct9
+tjJixYWICj3qHTx+eqDhueoFgmA/76o+eOEHLXUBBVZMPKS/aBoWIDJ5WHAKfiV3
+BFAa6zSgul6267YxEtHPC6y1gjuMvba0otJKppOeb3aGlzxNBw7+7EcJipZYAB/F
+BER+OIW+4qamSTKj1h7dbJ/PktahVw9N6Srf77ARAoIBAQDnoxp1/u7Er707mfzn
+ukDBfB7/ige3S+bdqftgj/Cb8BjJY6QB/F9XfJR79hOh539zwnlVLCdacMoKnwyq
+722ngpuRjlgPIc63Iihsln8RDVJ4s9zNzO0BuyootyUc3Cm/CXLdsMDpJkxUDaau
+D2x6o+6NBTeFeaJJqnDZhetotyHEjUKsXqgwl4NRF8e9wNYWbvGfSTxiLunJlTm1
+INnkTd7jV+9WedPP9PoNEibLdOAFyRZO3keje8s7G6+gj1BsWeLPJapnLIhgEuQ+
+ZmSbrpESohzoLEpGYqiwf/C0VSd4q38S3M6QAeEcIdC9JDCFGXq5JmwB9K5WWHhe
+tsMZAoIBAD0ovavzEBKfrQ0yXTT0W1PDXXPlpIcpD5iGMTmiZez/0GRaXi/kNDgk
+nk6KITgbGQ/O5xy0fUK2pDe9ccWLS8pTamjouoDgdfLl0P2rA3/uILecjbq3kGGx
+XGxTFc9wgrLSNDLk0v8CdyVRFy6/JFLm19Ya91EjxU7AxIL+z/uuZSU5+VaF/qAW
+73o7+SjPtj1h/qk6sejC1hZ/43+LzeOVXs88oFolFR3eyayyJkCsElXuOG0BO+ll
+50VA7OQrFZ1XpfD1vIVlhpogXU2QFe0l1NH00i6/l4xXzpWI79oMu5a6S/oWuIGg
+Qvy0mUIIWxegXy6aVnRb+iikOHgtnZU=
+-----END PRIVATE KEY-----

backend/app/main.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
+
+	log "github.com/go-pkgz/lgr"
+	flags "github.com/jessevdk/go-flags"
+
+	"github.com/umputun/remark/backend/app/cmd"
+)
+
+// Opts with all cli commands and flags
+type Opts struct {
+	ServerCmd  cmd.ServerCommand  `command:"server"`
+	ImportCmd  cmd.ImportCommand  `command:"import"`
+	BackupCmd  cmd.BackupCommand  `command:"backup"`
+	RestoreCmd cmd.RestoreCommand `command:"restore"`
+	AvatarCmd  cmd.AvatarCommand  `command:"avatar"`
+	CleanupCmd cmd.CleanupCommand `command:"cleanup"`
+
+	RemarkURL    string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
+	SharedSecret string `long:"secret" env:"SECRET" required:"true" description:"shared secret key"`
+
+	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42 %s\n", revision)
+
+	var opts Opts
+	p := flags.NewParser(&opts, flags.Default)
+	p.CommandHandler = func(command flags.Commander, args []string) error {
+		setupLog(opts.Dbg)
+		// commands implements CommonOptionsCommander to allow passing set of extra options defined for all commands
+		c := command.(cmd.CommonOptionsCommander)
+		c.SetCommon(cmd.CommonOpts{
+			RemarkURL:    opts.RemarkURL,
+			SharedSecret: opts.SharedSecret,
+			Revision:     revision,
+		})
+		err := c.Execute(args)
+		if err != nil {
+			log.Printf("[ERROR] failed with %+v", err)
+		}
+		return err
+	}
+
+	if _, err := p.Parse(); err != nil {
+		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
+			os.Exit(0)
+		} else {
+			os.Exit(1)
+		}
+	}
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces, log.CallerPkg, log.CallerIgnore("logger"))
+}
+
+// getDump reads runtime stack and returns as a string
+func getDump() string {
+	maxSize := 5 * 1024 * 1024
+	stacktrace := make([]byte, maxSize)
+	length := runtime.Stack(stacktrace, true)
+	if length > maxSize {
+		length = maxSize
+	}
+	return string(stacktrace[:length])
+}
+
+func init() {
+	// catch SIGQUIT and print stack traces
+	sigChan := make(chan os.Signal)
+	go func() {
+		for range sigChan {
+			log.Printf("[INFO] SIGQUIT detected, dump:\n%s", getDump())
+		}
+	}()
+	signal.Notify(sigChan, syscall.SIGQUIT)
+}

backend/app/main_test.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMain(t *testing.T) {
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
+		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.Nil(t, err)
+	}()
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		st := time.Now()
+		main()
+		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		wg.Done()
+	}()
+
+	time.Sleep(200 * time.Millisecond) // let server start
+
+	// send ping
+	resp, err := http.Get("http://localhost:18202/api/v1/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	wg.Wait()
+}
+
+func TestGetDump(t *testing.T) {
+	dump := getDump()
+	assert.True(t, strings.Contains(dump, "goroutine"))
+	assert.True(t, strings.Contains(dump, "[running]"))
+	assert.True(t, strings.Contains(dump, "backend/app/main.go"))
+	log.Printf("\n dump: %s", dump)
+}

backend/app/migrator/backup.go

@@ -5,12 +5,12 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 

backend/app/migrator/disqus.go

@@ -3,13 +3,13 @@ package migrator
 import (
 	"encoding/xml"
 	"io"
-	"log"
 	"strings"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
 )
 
 // Disqus implements Importer from disqus xml
@@ -68,14 +68,14 @@ func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
 	}
 
 	if failed > 0 {
-		return passed, errors.Errorf("failed to save %d comments", failed)
+		err = errors.Errorf("failed to save %d comments", failed)
+		if passed == 0 {
+			err = errors.New("import failed")
+		}
 	}
 
 	log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
 
-	if failed > 0 && passed == 0 {
-		err = errors.New("import failed")
-	}
 	return passed, err
 }
 
@@ -158,7 +158,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 	return commentsCh
 }
 
-func (d *Disqus) cleanText(text string) string {
+func (*Disqus) cleanText(text string) string {
 	text = strings.Replace(text, "\n", "", -1)
 	text = strings.Replace(text, "\t", "", -1)
 	return text

backend/app/migrator/disqus_test.go

@@ -6,37 +6,38 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestDisqus_Import(t *testing.T) {
 	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b}
+	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	d := Disqus{DataStore: &dataStore}
-	size, err := d.Import(strings.NewReader(xmlTest), "test")
+	size, err := d.Import(strings.NewReader(xmlTestDisqus), "test")
 	assert.Nil(t, err)
-	assert.Equal(t, 3, size)
+	assert.Equal(t, 4, size)
 
 	last, err := dataStore.Last("test", 10)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
+	assert.Equal(t, 4, len(last), "4 comments imported")
 
-	c := last[0]
+	c := last[len(last)-1] // last reverses, get first one
-	assert.True(t, strings.HasPrefix(c.Text, "<p>Google App Engine"))
+	assert.True(t, strings.HasPrefix(c.Text, "<p>The quick brown fox"))
-	assert.Equal(t, "299986072", c.ID)
+	assert.Equal(t, "299619020", c.ID)
 	assert.Equal(t, "", c.ParentID)
-	assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator)
+	assert.Equal(t, store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"}, c.Locator)
-	assert.Equal(t, "Dmitry Noname", c.User.Name)
+	assert.Equal(t, "Alexander Blah", c.User.Name)
-	assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID)
+	assert.Equal(t, "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434", c.User.ID)
-	assert.Equal(t, "96243f024cf6ad42b66f0c72709ae20b5d10ec14", c.User.IP)
+	assert.Equal(t, "2ba6b71dbf9750ae3356cce14cac6c1b1962747c", c.User.IP)
 
 	posts, err := dataStore.List("test", 0, 0)
 	assert.Nil(t, err)
@@ -49,13 +50,13 @@ func TestDisqus_Import(t *testing.T) {
 
 func TestDisqus_Convert(t *testing.T) {
 	d := Disqus{}
-	ch := d.convert(strings.NewReader(xmlTest), "test")
+	ch := d.convert(strings.NewReader(xmlTestDisqus), "test")
 
 	res := []store.Comment{}
 	for comment := range ch {
 		res = append(res, comment)
 	}
-	assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded")
+	assert.Equal(t, 4, len(res), "4 comments total, 1 spam excluded, 1 bad excluded")
 
 	exp0 := store.Comment{
 		ID: "299619020",
@@ -74,7 +75,7 @@ func TestDisqus_Convert(t *testing.T) {
 	assert.Equal(t, exp0, res[0])
 }
 
-var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
+var xmlTestDisqus = `<?xml version="1.0" encoding="utf-8"?>
 <disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
 
 	<category dsq:id="707279">
@@ -101,6 +102,7 @@ var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 		<isClosed>false</isClosed>
 		<isDeleted>false</isDeleted>
 	</thread>
+
 	<thread dsq:id="247937687">
 		<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
 		<forum>radiot</forum>
@@ -120,6 +122,7 @@ var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 		<isDeleted>false</isDeleted>
 	</thread>
 
+
 	<post dsq:id="299619020">
 		<id>3565798471341011339</id>
 		<message>
@@ -174,6 +177,23 @@ var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 		<thread dsq:id="247918464"/>
 	</post>
 
+	<post>
+		<id>12345678890</id>
+		<message>This comment had no ID</message>
+		<createdAt>2011-08-31T22:49:43Z</createdAt>
+		<forum>radiot</forum>
+		<isDeleted>false</isDeleted>
+		<isSpam>false</isSpam>
+		<author>
+			<email>blah.noname@gmail.com</email>
+			<name>Blah Noname</name>
+			<isAnonymous>false</isAnonymous>
+			<username>74b9e7568ef6860e93862c5d77590123</username>
+		</author>
+		<ipAddress>189.89.89.139</ipAddress>
+		<thread dsq:id="247918464"/>
+	</post>
+
 	<post dsq:id="299986073">
 		<id>6580890074280459219</id>
 		<message>some ugly spam</message>
@@ -189,5 +209,21 @@ var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
 		<ipAddress>189.89.89.139</ipAddress>
 		<thread dsq:id="247937687"/>
 	</post>
+
+	<post dsq:id="x299986073">
+		<message>some bad comment</message>
+		<createdAt>2011-x09-30T22:48:43Z</createdAt>
+		<isDeleted>false</isDeleted>
+		<isSpam>123</isSpam>
+		<author>
+			<email>noname@gmail.com</email>
+			<name>Noname</name>
+			<isAnonymous>true</isAnonymous>
+			<username>google-2c5d77590123</username>
+		</author>
+		<ipAddress>189.89.89.39</ipAddress>
+		<thread dsq:id=247937687/>
+	</post>
+
 </disqus>
 `

backend/app/migrator/migrator.go

@@ -5,11 +5,13 @@ package migrator
 
 import (
 	"io"
-	"log"
 	"os"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/store"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 // Importer defines interface to convert posts from external sources
@@ -28,6 +30,8 @@ type Store interface {
 	Find(locator store.Locator, sort string) ([]store.Comment, error)
 	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
 	DeleteAll(siteID string) error
+	Metas(siteID string) (umetas []service.UserMetaData, pmetas []service.PostMetaData, err error)
+	SetMetas(siteID string, umetas []service.UserMetaData, pmetas []service.PostMetaData) error
 }
 
 // ImportParams defines everything needed to run import
@@ -46,8 +50,10 @@ func ImportComments(p ImportParams) (int, error) {
 	switch p.Provider {
 	case "disqus":
 		importer = &Disqus{DataStore: p.DataStore}
+	case "wordpress":
+		importer = &WordPress{DataStore: p.DataStore}
 	case "native":
-		importer = &Remark{DataStore: p.DataStore}
+		importer = &Native{DataStore: p.DataStore}
 	default:
 		return 0, errors.Errorf("unsupported import provider %s", p.Provider)
 	}

backend/app/migrator/migrator_test.go

@@ -5,12 +5,13 @@ import (
 	"os"
 	"testing"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store/service"
 
-	"github.com/umputun/remark/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_ImportDisqus(t *testing.T) {
@@ -19,12 +20,12 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 		os.Remove("/tmp/disqus-test.xml")
 	}()
 
-	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600)
+	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTestDisqus), 0600)
 	require.Nil(t, err)
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
 		InputFile: "/tmp/disqus-test.xml",
@@ -32,6 +33,32 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 		Provider:  "disqus",
 	})
 	assert.Nil(t, err)
+	assert.Equal(t, 4, size)
+
+	last, err := dataStore.Last("test", 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 4, len(last), "4 comments imported")
+}
+
+func TestMigrator_ImportWordPress(t *testing.T) {
+	defer func() {
+		os.Remove("/tmp/remark-test.db")
+		os.Remove("/tmp/wordpress-test.xml")
+	}()
+
+	err := ioutil.WriteFile("/tmp/wordpress-test.xml", []byte(xmlTestWP), 0600)
+	require.Nil(t, err)
+
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
+	require.Nil(t, err, "create store")
+	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	size, err := ImportComments(ImportParams{
+		DataStore: dataStore,
+		InputFile: "/tmp/wordpress-test.xml",
+		SiteID:    "test",
+		Provider:  "wordpress",
+	})
+	assert.Nil(t, err)
 	assert.Equal(t, 3, size)
 
 	last, err := dataStore.Last("test", 10)
@@ -39,13 +66,13 @@ func TestMigrator_ImportDisqus(t *testing.T) {
 	assert.Equal(t, 3, len(last), "3 comments imported")
 }
 
-func TestMigrator_ImportRemark(t *testing.T) {
+func TestMigrator_ImportNative(t *testing.T) {
 	defer func() {
 		os.Remove("/tmp/remark-test.db")
 		os.Remove("/tmp/disqus-test.r42")
 	}()
 
-	data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
+	data := `{"version":1} {"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
 		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
 
 	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
@@ -53,7 +80,7 @@ func TestMigrator_ImportRemark(t *testing.T) {
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
 	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
+	dataStore := &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
 
 	size, err := ImportComments(ImportParams{
 		DataStore: dataStore,
@@ -70,7 +97,7 @@ func TestMigrator_ImportRemark(t *testing.T) {
 }
 
 func TestMigrator_ImportFailed(t *testing.T) {
-
+	defer os.Remove("/tmp/remark-test.db")
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
 	require.Nil(t, err, "create store")
 	dataStore := &service.DataStore{Interface: b}

backend/app/migrator/native.go

@@ -0,0 +1,138 @@
+package migrator
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+const natvieVersion = 1
+
+// Native implements exporter and importer for internal store format
+// {"version": 1, comments:[{...}\n,{}], meta: {meta}}
+// each comments starts from the new line
+type Native struct {
+	DataStore Store
+}
+
+type meta struct {
+	Version int                    `json:"version"`
+	Users   []service.UserMetaData `json:"users"`
+	Posts   []service.PostMetaData `json:"posts"`
+}
+
+// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
+// The final file is a valid json
+func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
+
+	if err = n.exportMeta(siteID, w); err != nil {
+		return 0, errors.Wrapf(err, "failed to export meta for site %s", siteID)
+	}
+
+	topics, err := n.DataStore.List(siteID, 0, 0)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Printf("[DEBUG] exporting %d topics", len(topics))
+	commentsCount := 0
+	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
+		topic := topics[i]
+		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
+		if err != nil {
+			return commentsCount, e
+		}
+
+		for _, comment := range comments {
+
+			buf := &bytes.Buffer{}
+			enc := json.NewEncoder(buf)
+			enc.SetEscapeHTML(false)
+
+			if err = enc.Encode(comment); err != nil {
+				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
+			}
+			if _, err = w.Write(buf.Bytes()); err != nil {
+				return commentsCount, errors.Wrap(err, "can't write comment data")
+			}
+			commentsCount++
+		}
+	}
+	log.Printf("[DEBUG] exported %d comments", commentsCount)
+	return commentsCount, nil
+}
+
+// exportMeta appends user and post metas to exported stream
+func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
+	m := meta{Version: natvieVersion}
+	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
+	if err != nil {
+		return errors.Wrap(err, "can't get meta")
+	}
+
+	if err := json.NewEncoder(w).Encode(m); err != nil {
+		return errors.Wrap(err, "can't encode meta")
+	}
+	return nil
+}
+
+// Import comments from json strings produced by Remark.Export
+func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
+
+	m := meta{}
+	dec := json.NewDecoder(reader)
+	if err = dec.Decode(&m); err != nil {
+		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
+	}
+
+	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+		return 0, errors.Errorf("unexpected import file version %d", m.Version)
+	}
+
+	if err = n.DataStore.DeleteAll(siteID); err != nil {
+		return 0, err
+	}
+
+	failed := 0
+	total, comments := 0, 0
+
+	for {
+		comment := store.Comment{}
+		err = dec.Decode(&comment)
+		if err == io.EOF {
+			break
+		}
+
+		total++
+
+		if err != nil {
+			failed++
+			continue
+		}
+
+		if _, err = n.DataStore.Create(comment); err != nil {
+			failed++
+			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
+			continue
+		}
+		comments++
+		if comments%1000 == 0 {
+			log.Printf("[DEBUG] imported %d comments", comments)
+		}
+	}
+
+	if failed > 0 {
+		return comments, errors.Errorf("failed to save %d comments", failed)
+	}
+	log.Printf("[INFO] imported %d comments from %d records", comments, total)
+
+	err = n.DataStore.SetMetas(siteID, m.Users, m.Posts)
+
+	return comments, err
+}

backend/app/migrator/native_test.go

@@ -0,0 +1,166 @@
+package migrator
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	log "github.com/go-pkgz/lgr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+var testDb = "/tmp/test-remark.db"
+
+func TestNative_Export(t *testing.T) {
+	defer os.Remove(testDb)
+	b := prep(t) // write 2 comments
+	assert.NoError(t, b.SetReadOnly(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, true))
+	assert.NoError(t, b.SetVerified("radio-t", "user1", true))
+	assert.NoError(t, b.SetBlock("radio-t", "user2", true, time.Hour))
+	r := Native{DataStore: b}
+
+	buf := &bytes.Buffer{}
+	size, err := r.Export(buf, "radio-t")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, size)
+
+	c1 := buf.String()
+	log.Print(c1)
+
+	dec := json.NewDecoder(strings.NewReader(c1))
+
+	meta := struct {
+		Version int                    `json:"version"`
+		Users   []service.UserMetaData `json:"users"`
+		Posts   []service.PostMetaData `json:"posts"`
+	}{}
+
+	require.NoError(t, dec.Decode(&meta), "decode meta")
+
+	assert.Equal(t, 2, len(meta.Users))
+	assert.Equal(t, "user1", meta.Users[0].ID)
+	assert.Equal(t, false, meta.Users[0].Blocked.Status)
+	assert.Equal(t, true, meta.Users[0].Verified)
+	assert.Equal(t, "user2", meta.Users[1].ID)
+	assert.Equal(t, true, meta.Users[1].Blocked.Status)
+	assert.Equal(t, false, meta.Users[1].Verified)
+
+	assert.Equal(t, 1, len(meta.Posts))
+	assert.Equal(t, "https://radio-t.com", meta.Posts[0].URL)
+	assert.Equal(t, true, meta.Posts[0].ReadOnly)
+
+	comments := [3]store.Comment{}
+
+	assert.NoError(t, dec.Decode(&comments[0]), "decode comment 0")
+	assert.NoError(t, dec.Decode(&comments[1]), "decode comment 0")
+	assert.Error(t, dec.Decode(&comments[2]), "EOF")
+
+	assert.Equal(t, "some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>", comments[0].Text)
+}
+
+func TestNative_Import(t *testing.T) {
+	defer os.Remove(testDb)
+
+	inp := `{"version":1,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	size, err := r.Import(strings.NewReader(inp), "radio-t")
+	assert.Nil(t, err)
+	assert.Equal(t, 2, size)
+
+	comments, err := b.Last("radio-t", 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments))
+	assert.Equal(t, "f863bd79-fec6-4a75-b308-61fe5dd02aa1", comments[0].ID)
+	assert.Equal(t, "1234", comments[0].ParentID)
+	assert.Equal(t, false, b.IsReadOnly(comments[0].Locator))
+
+	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
+	assert.Equal(t, "https://radio-t.com", comments[1].Locator.URL)
+	assert.Equal(t, true, b.IsReadOnly(comments[1].Locator))
+
+	assert.Equal(t, false, b.IsBlocked("radio-t", "user1"))
+	assert.Equal(t, true, b.IsVerified("radio-t", "user1"))
+
+	assert.Equal(t, true, b.IsBlocked("radio-t", "user2"))
+	assert.Equal(t, false, b.IsVerified("radio-t", "user2"))
+}
+
+func TestNative_ImportWrongVersion(t *testing.T) {
+	inp := `{"version":2,"users":[{"id":"user1","blocked":{"status":false,"until":"0001-01-01T00:00:00Z"},"verified":true},{"id":"user2","blocked":{"status":true,"until":"2018-12-23T02:55:22.472041-06:00"},"verified":false}],"posts":[{"url":"https://radio-t.com","read_only":true}]}
+	{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}
+	{"id":"f863bd79-fec6-4a75-b308-61fe5dd02aa1","pid":"1234","text":"some text2","user":{"name":"user name","id":"user2","picture":"","ip":"293ec5b0cf154855258824ec7fac5dc63d176915","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com/2"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}`
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	size, err := r.Import(strings.NewReader(inp), "radio-t")
+	assert.EqualError(t, err, "unexpected import file version 2")
+	assert.Equal(t, 0, size)
+
+}
+func TestNative_ImportManyWithError(t *testing.T) {
+	defer os.Remove(testDb)
+
+	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
+
+	buf := &bytes.Buffer{}
+	buf.WriteString(`{"version":1, "users":[], "posts":[]}` + "\n")
+	for i := 0; i < 1200; i++ {
+		buf.WriteString(fmt.Sprintf(goodRec, i))
+	}
+	buf.WriteString("{}\n")
+	buf.WriteString("{}\n")
+
+	b := prep(t) // write some recs
+	r := Native{DataStore: &service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}}
+	n, err := r.Import(buf, "radio-t")
+	assert.EqualError(t, err, "failed to save 2 comments")
+	assert.Equal(t, 1200, n)
+	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
+	assert.Nil(t, err)
+	assert.Equal(t, 1200, len(comments))
+}
+
+// makes new boltdb, put two records
+func prep(t *testing.T) *service.DataStore {
+	os.Remove(testDb)
+
+	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
+	assert.Nil(t, err)
+
+	b := &service.DataStore{Interface: boltStore, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+
+	comment := store.Comment{
+		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	comment = store.Comment{
+		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:    store.User{ID: "user2", Name: "user name"},
+	}
+	_, err = b.Create(comment)
+	assert.Nil(t, err)
+
+	return b
+}

backend/app/migrator/wordpress.go

@@ -0,0 +1,156 @@
+package migrator
+
+import (
+	"encoding/xml"
+	"html"
+	"io"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+const wpTimeLayout = "2006-01-02 15:04:05"
+
+// WordPress implements Importer from WP xml
+type WordPress struct {
+	DataStore Store
+}
+
+type wpItem struct {
+	Link     string      `xml:"link"`
+	Comments []wpComment `xml:"comment"`
+}
+
+type wpComment struct {
+	ID          string `xml:"comment_id"`
+	Author      string `xml:"comment_author"`
+	AuthorEmail string `xml:"comment_author_email"`
+	AuthorIP    string `xml:"comment_author_IP"`
+	Date        wpTime `xml:"comment_date_gmt"`
+	Content     string `xml:"comment_content"`
+	Approved    string `xml:"comment_approved"`
+	PID         string `xml:"comment_parent"`
+}
+
+type wpTime struct {
+	time time.Time
+}
+
+func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var v string
+	if err := d.DecodeElement(&v, &start); err != nil {
+		return err
+	}
+	t, err := time.Parse(wpTimeLayout, v)
+	if err != nil {
+		return err
+	}
+	w.time = t
+	return err
+}
+
+// Convert satisfies formatter.CommentConverter
+func (w *WordPress) Convert(text string) string {
+	return html.UnescapeString(text) // sanitize remains on comment create
+}
+
+// Import comments from WP and save to store
+func (w *WordPress) Import(r io.Reader, siteID string) (size int, err error) {
+
+	if err = w.DataStore.DeleteAll(siteID); err != nil {
+		return 0, err
+	}
+
+	commentsCh := w.convert(r, siteID)
+	failed, passed := 0, 0
+	for c := range commentsCh {
+		if _, err = w.DataStore.Create(c); err != nil {
+			failed++
+			continue
+		}
+		passed++
+	}
+
+	if failed > 0 {
+		err = errors.Errorf("failed to save %d comments", failed)
+		if passed == 0 {
+			err = errors.New("import failed")
+		}
+	}
+
+	log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
+
+	return passed, err
+}
+
+func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
+
+	decoder := xml.NewDecoder(r)
+	commentsCh := make(chan store.Comment)
+
+	stats := struct {
+		inpItems, failedItems       int
+		inpComments, failedComments int
+		rejectedComments            int // not approved
+	}{}
+
+	commentFormatter := store.NewCommentFormatter(w)
+
+	go func() {
+		for {
+			t, err := decoder.Token()
+			if t == nil || err != nil {
+				break
+			}
+
+			switch el := t.(type) {
+			case xml.StartElement:
+				if el.Name.Local == "item" {
+					stats.inpItems++
+					item := wpItem{}
+					if err := decoder.DecodeElement(&item, &el); err != nil {
+						log.Printf("[WARN] Can't decode item, %s", err)
+						stats.failedItems++
+						continue
+					}
+					if item.Comments != nil {
+						for _, comment := range item.Comments {
+							if comment.Approved != "1" {
+								stats.rejectedComments++
+								continue
+							}
+
+							if comment.PID == "0" {
+								comment.PID = ""
+							}
+
+							c := store.Comment{
+								ID:      comment.ID,
+								Locator: store.Locator{URL: item.Link, SiteID: siteID},
+								User: store.User{
+									ID:   "wordpress_" + store.EncodeID(comment.Author),
+									Name: comment.Author,
+									IP:   comment.AuthorIP,
+								},
+								Text:      comment.Content,
+								Timestamp: comment.Date.time,
+								ParentID:  comment.PID,
+							}
+							commentsCh <- commentFormatter.Format(c)
+							stats.inpComments++
+							if stats.inpComments%1000 == 0 {
+								log.Printf("[DEBUG] proccessed %d comments", stats.inpComments)
+							}
+						}
+					}
+				}
+			}
+		}
+		close(commentsCh)
+		log.Printf("[INFO] converted %d comments, %+v", stats.inpComments-stats.failedComments, stats)
+	}()
+	return commentsCh
+}

backend/app/migrator/wordpress_test.go

@@ -0,0 +1,322 @@
+package migrator
+
+import (
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/bbolt"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+func TestWordPress_Import(t *testing.T) {
+	siteID := "testWP"
+	defer os.Remove("/tmp/remark-test.db")
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: siteID})
+	assert.Nil(t, err, "create store")
+
+	dataStore := service.DataStore{Interface: b, AdminStore: admin.NewStaticStore("12345", []string{}, "")}
+	wp := WordPress{DataStore: &dataStore}
+	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, size)
+
+	last, err := dataStore.Last(siteID, 10)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(last), "3 comments imported")
+
+	c := last[0]
+	assert.Equal(t, "14", c.ID)
+	assert.Equal(t, store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID}, c.Locator)
+	assert.Equal(t, "wordpress_75b2b81081f82495d7af26759e67af6554ffda4a", c.User.ID)
+	assert.Equal(t, "SuperUser3", c.User.Name)
+	assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
+	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
+	assert.Equal(t, ts, c.Timestamp)
+	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+
+	posts, err := dataStore.List(siteID, 0, 0)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(posts))
+
+	p := posts[0]
+	assert.Equal(t, "https://realmenweardress.es/2010/07/do-you-rp/", p.URL)
+
+	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
+	assert.Nil(t, err)
+	assert.Equal(t, 3, count)
+}
+
+func TestWordPress_Convert(t *testing.T) {
+	wp := WordPress{}
+	ch := wp.convert(strings.NewReader(xmlTestWP), "testWP")
+
+	comments := []store.Comment{}
+	for c := range ch {
+		comments = append(comments, c)
+	}
+	assert.Equal(t, 3, len(comments), "3 comments exported, 1 excluded")
+
+	exp1 := store.Comment{
+		ID: "13",
+		Locator: store.Locator{
+			SiteID: "testWP",
+			URL:    "https://realmenweardress.es/2010/07/do-you-rp/",
+		},
+		Text: `<p>[…] I know I’m a bit loony with my attachment to my bankers.  I’m glad I’m not the only one. […]</p>` + "\n",
+		User: store.User{
+			Name: "Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual",
+			ID:   "wordpress_" + store.EncodeID("Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual"),
+			IP:   "74.200.244.101",
+		},
+	}
+	exp1.Timestamp, _ = time.Parse(wpTimeLayout, "2010-07-21 14:02:08")
+	assert.Equal(t, exp1, comments[1])
+}
+
+func TestWP_Convert_MD(t *testing.T) {
+	wp := WordPress{}
+	ch := wp.convert(strings.NewReader(xmlTestWPmd), "siteID")
+
+	comments := []store.Comment{}
+	for c := range ch {
+		comments = append(comments, c)
+	}
+	assert.Equal(t, 3, len(comments), "3 comments exported")
+
+	assert.Equal(t, "<p>Row1<br/>\nRow2</p>\n\n<p>Row4</p>\n", comments[0].Text)
+
+	assert.Equal(t, "<p>markdown <code>text</code></p>\n", comments[1].Text)
+
+	expText := `<p>Row1 Link <a href="http://releases.rancher.com/os/latest">http://releases.rancher.com/os/latest</a> markdown <code>text</code> blah</p>`
+	expText += "\n\n<p>Row3 markdown<code>md block</code></p>\n"
+	assert.Equal(t, expText, comments[2].Text)
+}
+
+var xmlTestWP = `
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0"
+	xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/"
+	xmlns:content="http://purl.org/rss/1.0/modules/content/"
+	xmlns:wfw="http://wellformedweb.org/CommentAPI/"
+	xmlns:dc="http://purl.org/dc/elements/1.1/"
+	xmlns:wp="http://wordpress.org/export/1.2/"
+>
+
+<channel>
+	<title>Real Men Wear Dress.es</title>
+	<link>https://realmenweardress.es</link>
+	<description>SuperAdmin&#039;s gaming and technological musings</description>
+	<pubDate>Mon, 23 Jul 2018 10:21:47 +0000</pubDate>
+	<language>en-US</language>
+	<wp:wxr_version>1.2</wp:wxr_version>
+	<wp:base_site_url>https://realmenweardress.es</wp:base_site_url>
+	<wp:base_blog_url>https://realmenweardress.es</wp:base_blog_url>
+
+	<wp:author><wp:author_id>2</wp:author_id><wp:author_login><![CDATA[SuperAdmin]]></wp:author_login><wp:author_email><![CDATA[superadmin@super.eu]]></wp:author_email><wp:author_display_name><![CDATA[SuperAdmin]]></wp:author_display_name><wp:author_first_name><![CDATA[SuperAdmin]]></wp:author_first_name><wp:author_last_name><![CDATA[superadmin]]></wp:author_last_name></wp:author>
+	<wp:author><wp:author_id>1</wp:author_id><wp:author_login><![CDATA[admin]]></wp:author_login><wp:author_email><![CDATA[superadmin@superadmin.co.uk]]></wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
+
+	<wp:category>
+		<wp:term_id>25</wp:term_id>
+		<wp:category_nicename><![CDATA[cataclysm]]></wp:category_nicename>
+		<wp:category_parent><![CDATA[]]></wp:category_parent>
+		<wp:cat_name><![CDATA[Cataclysm]]></wp:cat_name>
+	</wp:category>
+
+	<wp:tag>
+		<wp:term_id>39</wp:term_id>
+		<wp:tag_slug><![CDATA[addons]]></wp:tag_slug>
+		<wp:tag_name><![CDATA[addons]]></wp:tag_name>
+	</wp:tag>
+
+	<generator>https://wordpress.org/?v=4.8.1</generator>
+
+	<item>
+		<title>Post without comments</title>
+		<link>https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/</link>
+		<pubDate>Sat, 19 Jun 2010 08:34:13 +0000</pubDate>
+		<dc:creator><![CDATA[admin]]></dc:creator>
+		<guid isPermaLink="false">http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg</guid>
+		<description></description>
+		<content:encoded><![CDATA[So you can actually fly into the well it appears and if your lucky you stay mounted. I imagine it terrifies the poor rats.]]></content:encoded>
+		<excerpt:encoded><![CDATA[]]></excerpt:encoded>
+		<wp:post_id>6</wp:post_id>
+		<wp:post_date><![CDATA[2010-06-19 08:34:13]]></wp:post_date>
+		<wp:post_date_gmt><![CDATA[2010-06-19 08:34:13]]></wp:post_date_gmt>
+		<wp:comment_status><![CDATA[open]]></wp:comment_status>
+		<wp:ping_status><![CDATA[open]]></wp:ping_status>
+		<wp:post_name><![CDATA[screenshot_013110_200413]]></wp:post_name>
+		<wp:status><![CDATA[inherit]]></wp:status>
+		<wp:post_parent>1</wp:post_parent>
+		<wp:menu_order>0</wp:menu_order>
+		<wp:post_type><![CDATA[attachment]]></wp:post_type>
+		<wp:post_password><![CDATA[]]></wp:post_password>
+		<wp:is_sticky>0</wp:is_sticky>
+		<wp:attachment_url><![CDATA[https://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:attachment_url>
+		<wp:postmeta>
+			<wp:meta_key><![CDATA[_wp_attached_file]]></wp:meta_key>
+			<wp:meta_value><![CDATA[2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:meta_value>
+		</wp:postmeta>
+	</item>
+	<item>
+		<title>Post with comments. One is not approved</title>
+		<link>https://realmenweardress.es/2010/07/do-you-rp/</link>
+		<pubDate>Mon, 19 Jul 2010 14:24:22 +0000</pubDate>
+		<dc:creator><![CDATA[SuperAdmin]]></dc:creator>
+		<guid isPermaLink="false">http://realmenweardress.es/?p=100</guid>
+		<description></description>
+		<content:encoded><![CDATA[<a href="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546180886.jpeg"><img class="size-thumbnail wp-image-102 alignleft" title="I need to stand on things else I can't reach" src="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546270587-120x120.jpg" alt="I need to stand on things else I can't reach" width="120" height="120" /></a>Meet Grokknomel?]]></content:encoded>
+		<excerpt:encoded><![CDATA[]]></excerpt:encoded>
+		<wp:post_id>100</wp:post_id>
+		<wp:post_date><![CDATA[2010-07-19 14:24:22]]></wp:post_date>
+		<wp:post_date_gmt><![CDATA[2010-07-19 14:24:22]]></wp:post_date_gmt>
+		<wp:comment_status><![CDATA[open]]></wp:comment_status>
+		<wp:ping_status><![CDATA[open]]></wp:ping_status>
+		<wp:post_name><![CDATA[do-you-rp]]></wp:post_name>
+		<wp:status><![CDATA[publish]]></wp:status>
+		<wp:post_parent>0</wp:post_parent>
+		<wp:menu_order>0</wp:menu_order>
+		<wp:post_type><![CDATA[post]]></wp:post_type>
+		<wp:post_password><![CDATA[]]></wp:post_password>
+		<wp:is_sticky>0</wp:is_sticky>
+		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
+		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
+		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
+		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<wp:postmeta>
+			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
+			<wp:meta_value><![CDATA[2]]></wp:meta_value>
+		</wp:postmeta>
+		<wp:comment>
+			<wp:comment_id>8</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser1]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[superuser1@aol.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://superuser1.blogspot.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[79.141.141.73]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-20 12:08:08]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-20 12:08:08]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[I do catch myself]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>9</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser2]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[superuser2@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://thewowstorm.wordpress.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[97.36.113.1]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-20 13:09:25]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-20 13:09:25]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[I think it us inherent in the game to start seeing your character as a personality]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[0]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>13</wp:comment_id>
+			<wp:comment_author><![CDATA[Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[]]></wp:comment_author_email>
+			<wp:comment_author_url>http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[74.200.244.101]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-21 14:02:08]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-21 14:02:08]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[[...] I know I&#8217;m a bit loony with my attachment to my bankers.  I&#8217;m glad I&#8217;m not the only one. [...]]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[pingback]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>14</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser3]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[blablah@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://realmenweardress.es</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>13</wp:comment_parent>
+			<wp:comment_user_id>2</wp:comment_user_id>
+		</wp:comment>
+	</item>
+	</channel>
+</rss>
+`
+
+// parts of unused xml tags are omitted
+var xmlTestWPmd = `
+<?xml version="1.0" encoding="UTF-8" ?>
+<channel>
+	<item>
+		<title>Deploying RancherOS on Vultr instances</title>
+		<link>https://realmenweardress.es/2016/07/deploying-rancheros-on-vultr-instances/</link>
+
+		<wp:comment>
+			<wp:comment_id>1</wp:comment_id>
+			<wp:comment_author><![CDATA[user1]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>https://eric.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[Row1
+Row2
+
+Row4]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+
+		<wp:comment>
+			<wp:comment_id>2</wp:comment_id>
+			<wp:comment_author><![CDATA[user1]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>https://eric.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[markdown ` + "`" + "text" + "`" + `]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+
+		<wp:comment>
+			<wp:comment_id>2</wp:comment_id>
+			<wp:comment_author><![CDATA[user1]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[eric@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>https://eric.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[96.54.240.57]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2017-12-11 00:08:56]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2017-12-11 00:08:56]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[Row1 Link http://releases.rancher.com/os/latest markdown ` + "`" + "text" + "`" + ` blah
+
+Row3 markdown` +
+	"```" +
+	"md block" +
+	"```" +
+	`]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+
+	</item>
+</channel>
+</rss>
+`

backend/app/notify/notify.go

@@ -0,0 +1,111 @@
+// Package notify provides notification functionality.
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// Service delivers notifications to multiple destinations
+type Service struct {
+	dataService  Store
+	destinations []Destination
+	queue        chan request
+
+	closed uint32 // non-zero means closed. uses uint instead of bool for atomic
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+// Destination defines interface for a given destination service, like telegram, email and so on
+type Destination interface {
+	fmt.Stringer
+	Send(ctx context.Context, req request) error
+}
+
+// Store defines the minimal interface accessing stored commens used by notifier
+type Store interface {
+	Get(locator store.Locator, id string) (store.Comment, error)
+}
+type request struct {
+	comment store.Comment
+	parent  store.Comment
+}
+
+const defaultQueueSize = 100
+const uiNav = "#remark42__comment-"
+
+// NewService makes notification service routing comments to all destinations.
+func NewService(dataService Store, size int, destinations ...Destination) *Service {
+	if size <= 0 {
+		size = defaultQueueSize
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	res := Service{
+		dataService:  dataService,
+		queue:        make(chan request, size),
+		destinations: destinations,
+		ctx:          ctx,
+		cancel:       cancel,
+	}
+	if len(destinations) > 0 {
+		go res.do()
+	}
+	log.Printf("[INFO] create notifier service, queue size=%d, destinations=%d", size, len(destinations))
+	return &res
+}
+
+// Submit comment to internal channel if not busy, drop if can't send
+func (s *Service) Submit(comment store.Comment) {
+	if len(s.destinations) == 0 || atomic.LoadUint32(&s.closed) != 0 {
+		return
+	}
+	parentComment := store.Comment{}
+	if s.dataService != nil {
+		if p, err := s.dataService.Get(comment.Locator, comment.ParentID); err == nil {
+			parentComment = p
+		}
+	}
+	select {
+	case s.queue <- request{comment: comment, parent: parentComment}:
+	default:
+		log.Printf("[WARN] can't send comment notification to queue, %+v", comment)
+	}
+}
+
+// Close queue channel and wait for completion
+func (s *Service) Close() {
+	if s.queue != nil {
+		log.Print("[DEBUG] close notifier")
+		close(s.queue)
+		s.cancel()
+		<-s.ctx.Done()
+	}
+	atomic.StoreUint32(&s.closed, 1)
+}
+
+func (s *Service) do() {
+	for c := range s.queue {
+		var wg sync.WaitGroup
+		wg.Add(len(s.destinations))
+		for _, dest := range s.destinations {
+			go func(d Destination) {
+				if err := d.Send(s.ctx, c); err != nil {
+					log.Printf("[WARN] failed to send to %s, %s", d, err)
+				}
+				wg.Done()
+			}(dest)
+		}
+		wg.Wait()
+	}
+	log.Print("[WARN] terminated notifier")
+}
+
+// NopService is do-nothing notifier, without destinations
+var NopService = &Service{}

backend/app/notify/notify_test.go

@@ -0,0 +1,155 @@
+package notify
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestService_NoDestinations(t *testing.T) {
+	s := NewService(nil, 1)
+	assert.NotNil(t, s)
+	s.Submit(store.Comment{ID: "123"})
+	s.Submit(store.Comment{ID: "123"})
+	s.Submit(store.Comment{ID: "123"})
+	s.Close()
+}
+
+func TestService_WithDestinations(t *testing.T) {
+	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	s := NewService(nil, 1, d1, d2)
+	assert.NotNil(t, s)
+
+	s.Submit(store.Comment{ID: "100"})
+	time.Sleep(time.Millisecond * 110)
+	s.Submit(store.Comment{ID: "101"})
+	time.Sleep(time.Millisecond * 110)
+	s.Submit(store.Comment{ID: "102"})
+	time.Sleep(time.Millisecond * 110)
+	s.Close()
+
+	assert.Equal(t, 3, len(d1.get()), "got all comments to d1")
+	assert.Equal(t, 3, len(d2.get()), "got all comments to d2")
+
+	assert.Equal(t, "100", d1.get()[0].comment.ID)
+	assert.Equal(t, "101", d1.get()[1].comment.ID)
+	assert.Equal(t, "102", d1.get()[2].comment.ID)
+}
+
+func TestService_WithDrops(t *testing.T) {
+	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	s := NewService(nil, 1, d1, d2)
+	assert.NotNil(t, s)
+
+	s.Submit(store.Comment{ID: "100"})
+	s.Submit(store.Comment{ID: "101"})
+	time.Sleep(time.Millisecond * 110)
+	s.Submit(store.Comment{ID: "102"})
+	time.Sleep(time.Millisecond * 110)
+	s.Close()
+
+	s.Submit(store.Comment{ID: "111"}) // safe to send after close
+
+	assert.Equal(t, 2, len(d1.get()), "one comment dropped from d1")
+	assert.Equal(t, 2, len(d2.get()), "one comment dropped from d2")
+}
+
+func TestService_Many(t *testing.T) {
+	d1, d2 := &mockDest{id: 1}, &mockDest{id: 2}
+	s := NewService(nil, 5, d1, d2)
+	assert.NotNil(t, s)
+
+	for i := 0; i < 10; i++ {
+		s.Submit(store.Comment{ID: fmt.Sprintf("%d", 100+i)})
+		time.Sleep(time.Millisecond * time.Duration(rand.Int31n(200)))
+	}
+	s.Close()
+	time.Sleep(time.Millisecond * 10)
+
+	assert.NotEqual(t, 10, len(d1.get()), "some comments dropped from d1")
+	assert.NotEqual(t, 10, len(d2.get()), "some comments dropped from d2")
+
+	assert.True(t, d1.closed)
+	assert.True(t, d2.closed)
+}
+
+func TestService_WithParent(t *testing.T) {
+	dest := &mockDest{id: 1}
+	dataStore := &mockStore{data: map[string]store.Comment{}}
+
+	dataStore.data["p1"] = store.Comment{ID: "p1"}
+	dataStore.data["p2"] = store.Comment{ID: "p2"}
+
+	s := NewService(dataStore, 1, dest)
+	assert.NotNil(t, s)
+
+	s.Submit(store.Comment{ID: "c1", ParentID: "p1"})
+	time.Sleep(time.Millisecond * 110)
+	s.Submit(store.Comment{ID: "c11", ParentID: "p11"})
+	time.Sleep(time.Millisecond * 110)
+	s.Close()
+
+	destRes := dest.get()
+	assert.Equal(t, 2, len(destRes), "two comment notified")
+	assert.Equal(t, "p1", destRes[0].comment.ParentID)
+	assert.Equal(t, "p1", destRes[0].parent.ID)
+	assert.Equal(t, "p11", destRes[1].comment.ParentID)
+	assert.Equal(t, "", destRes[1].parent.ID)
+}
+
+func TestService_Nop(t *testing.T) {
+	s := NopService
+	s.Submit(store.Comment{})
+	s.Close()
+	assert.Equal(t, uint32(1), atomic.LoadUint32(&s.closed))
+}
+
+type mockDest struct {
+	data   []request
+	id     int
+	closed bool
+	lock   sync.Mutex
+}
+
+func (m *mockDest) Send(ctx context.Context, r request) error {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	select {
+	case <-time.After(100 * time.Millisecond):
+		m.data = append(m.data, r)
+		log.Printf("sent %s -> %d", r.comment.ID, m.id)
+	case <-ctx.Done():
+		log.Printf("ctx closed %d", m.id)
+		m.closed = true
+	}
+	return nil
+}
+
+func (m *mockDest) get() []request {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	res := make([]request, len(m.data))
+	copy(res, m.data)
+	return res
+}
+func (m *mockDest) String() string { return fmt.Sprintf("mock id=%d, closed=%v", m.id, m.closed) }
+
+type mockStore struct{ data map[string]store.Comment }
+
+func (m *mockStore) Get(_ store.Locator, id string) (store.Comment, error) {
+	res, ok := m.data[id]
+	if !ok {
+		return store.Comment{}, errors.New("no such id")
+	}
+	return res, nil
+}

backend/app/notify/telegram.go

@@ -0,0 +1,145 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"html"
+	"net/http"
+	"strconv"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
+	"github.com/pkg/errors"
+)
+
+// Telegram implements notify.Destination for telegram
+type Telegram struct {
+	channelID string // unique identifier for the target chat or username of the target channel (in the format @channelusername)
+	token     string
+	apiPrefix string
+	timeout   time.Duration
+}
+
+const telegramTimeOut = 5000 * time.Millisecond
+const telegramAPIPrefix = "https://api.telegram.org/bot"
+
+// NewTelegram makes telegram bot for notifications
+func NewTelegram(token string, channelID string, timeout time.Duration, api string) (*Telegram, error) {
+
+	if _, err := strconv.ParseInt(channelID, 10, 64); err != nil {
+		channelID = "@" + channelID // if channelID not a number enforce @ prefix
+	}
+
+	res := Telegram{channelID: channelID, token: token, apiPrefix: api, timeout: timeout}
+	if res.apiPrefix == "" {
+		res.apiPrefix = telegramAPIPrefix
+	}
+	if res.timeout == 0 {
+		res.timeout = telegramTimeOut
+	}
+	log.Printf("[DEBUG] create new telegram notifier for cham %s, timeout=%s, api=%s", channelID, res.timeout, res.timeout)
+
+	err := repeater.NewDefault(5, time.Millisecond*250).Do(func() error {
+		client := http.Client{Timeout: telegramTimeOut}
+		resp, err := client.Get(fmt.Sprintf("%s%s/getMe", res.apiPrefix, token))
+		if err != nil {
+			return errors.Wrap(err, "can't initialize telegram notifications")
+		}
+		defer func() {
+			if err = resp.Body.Close(); err != nil {
+				log.Printf("[WARN] can't close request body, %s", err)
+			}
+		}()
+
+		if resp.StatusCode != http.StatusOK {
+			return errors.Errorf("unexpected telegram status code %d", resp.StatusCode)
+		}
+
+		tgResp := struct {
+			OK     bool `json:"ok"`
+			Result struct {
+				FirstName string `json:"first_name"`
+				ID        uint64 `json:"id"`
+				IsBot     bool   `json:"is_bot"`
+				UserName  string `json:"username"`
+			}
+		}{}
+
+		if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
+			return errors.Wrap(err, "can't decode response")
+		}
+
+		if !tgResp.OK || !tgResp.Result.IsBot {
+			return errors.Errorf("unexpected telegram response %+v", tgResp)
+		}
+		return nil
+	})
+
+	return &res, err
+}
+
+// Send to telegram channel
+func (t *Telegram) Send(ctx context.Context, req request) error {
+	client := http.Client{Timeout: telegramTimeOut}
+	log.Printf("[DEBUG] send telegram notification to %s, comment id %s", t.channelID, req.comment.ID)
+
+	from := req.comment.User.Name
+	if req.comment.ParentID != "" {
+		from += " → " + req.parent.User.Name
+	}
+	from = "*" + from + "*"
+	link := fmt.Sprintf("↦ [original comment](%s)", req.comment.Locator.URL+uiNav+req.comment.ID)
+	if req.comment.PostTitle != "" {
+		link = fmt.Sprintf("↦ [%s](%s)", req.comment.PostTitle, req.comment.Locator.URL+uiNav+req.comment.ID)
+	}
+	u := fmt.Sprintf("%s%s/sendMessage?chat_id=%s&parse_mode=Markdown&disable_web_page_preview=true",
+		t.apiPrefix, t.token, t.channelID)
+
+	msg := fmt.Sprintf("%s\n\n%s\n\n%s", from, req.comment.Orig, link)
+	msg = html.UnescapeString(msg)
+	body := struct {
+		Text string `json:"text"`
+	}{Text: msg}
+
+	b, err := json.Marshal(body)
+	if err != nil {
+		return errors.Wrap(err, "failed to make telegram body")
+	}
+
+	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
+	if err != nil {
+		return errors.Wrap(err, "failed to make telegram request")
+	}
+
+	r = r.WithContext(ctx)
+	resp, err := client.Do(r)
+	if err != nil {
+		return errors.Wrap(err, "failed to get telegram response")
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] can't close request body, %s", err)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("unexpected telegram status code %d for url %q", resp.StatusCode, u)
+	}
+
+	tgResp := struct {
+		OK bool `json:"ok"`
+	}{}
+
+	if err = json.NewDecoder(resp.Body).Decode(&tgResp); err != nil {
+		return errors.Wrap(err, "can't decode telegram response")
+	}
+	return nil
+}
+
+func (t *Telegram) String() string {
+	return "telegram: " + t.channelID
+}

backend/app/notify/telegram_test.go

@@ -0,0 +1,120 @@
+package notify
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestTelegram_New(t *testing.T) {
+
+	ts := mockTelegramServer()
+	defer ts.Close()
+
+	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NoError(t, err)
+	assert.NotNil(t, tb)
+	assert.Equal(t, "@remark_test", tb.channelID, "@ added")
+
+	st := time.Now()
+	_, err = NewTelegram("bad-resp", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.EqualError(t, err, "unexpected telegram response {OK:false Result:{FirstName:comments_test ID:707381019 IsBot:false UserName:remark42_test_bot}}")
+	assert.True(t, time.Since(st) >= 250*5*time.Millisecond)
+
+	_, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "can't decode response:")
+
+	_, err = NewTelegram("404", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.EqualError(t, err, "unexpected telegram status code 404")
+
+	_, err = NewTelegram("no-such-thing", "remark_test", 2*time.Second, "http://127.0.0.1:4321/")
+	assert.EqualError(t, err, "can't initialize telegram notifications: Get http://127.0.0.1:4321/no-such-thing/getMe: dial tcp 127.0.0.1:4321: connect: connection refused")
+
+	_, err = NewTelegram("good-token", "remark_test", 2*time.Second, "")
+	assert.Error(t, err, "empty api url not allowed")
+
+	_, err = NewTelegram("good-token", "remark_test", 0, ts.URL+"/")
+	assert.NoError(t, err, "0 timeout allowed as default")
+
+	tb, err = NewTelegram("good-token", "1234567890", 2*time.Second, ts.URL+"/")
+	assert.NoError(t, err)
+	assert.NotNil(t, tb)
+	assert.Equal(t, "1234567890", tb.channelID, "no @ prefix")
+}
+
+func TestTelegram_Send(t *testing.T) {
+	ts := mockTelegramServer()
+	defer ts.Close()
+
+	tb, err := NewTelegram("good-token", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NoError(t, err)
+	assert.NotNil(t, tb)
+	c := store.Comment{Text: "some text", ParentID: "1"}
+	c.User.Name = "from"
+	cp := store.Comment{Text: "some parent text"}
+	cp.User.Name = "to"
+
+	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	assert.NoError(t, err)
+	c.PostTitle = "test title"
+	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	assert.NoError(t, err)
+
+	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
+	assert.NotNil(t, err, "should failed")
+	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
+
+	assert.Equal(t, "telegram: @remark_test", tb.String())
+}
+
+func mockTelegramServer() *httptest.Server {
+	router := chi.NewRouter()
+	router.Get("/good-token/getMe", func(w http.ResponseWriter, r *http.Request) {
+		s := `{"ok": true,
+				"result": {
+					"first_name": "comments_test",
+					"id": 707381019,
+					"is_bot": true,
+					"username": "remark42_test_bot"
+				}}`
+		_, _ = w.Write([]byte(s))
+	})
+	router.Get("/bad-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
+		s := `{"ok": false,
+				"result": {
+					"first_name": "comments_test",
+					"id": 707381019,
+					"is_bot": false,
+					"username": "remark42_test_bot"
+				}}`
+		_, _ = w.Write([]byte(s))
+	})
+	router.Get("/non-json-resp/getMe", func(w http.ResponseWriter, r *http.Request) {
+		s := `"ok": false,
+				"result": {
+					"first_name": "comments_test",
+					"id": 707381019,
+					"is_bot": false,
+					"username": "remark42_test_bot"
+				`
+		_, _ = w.Write([]byte(s))
+	})
+	router.Get("/404/getMe", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	})
+
+	router.Post("/good-token/sendMessage", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"ok": true}`))
+	})
+
+	return httptest.NewServer(router)
+}

backend/app/rest/api/admin.go

@@ -1,32 +1,30 @@
 package api
 
 import (
-	"compress/gzip"
 	"errors"
-	"fmt"
-	"io"
-	"log"
 	"net/http"
+	"path"
 	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	"github.com/umputun/remark/app/rest/auth"
+	"github.com/go-pkgz/auth"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 
-	"github.com/umputun/remark/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/app/rest"
+	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/app/rest/cache"
+	"github.com/umputun/remark/backend/app/store/service"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
 )
 
 // admin provides router for all requests available for admin users only
 type admin struct {
 	dataService   *service.DataStore
-	exporter      migrator.Exporter
 	cache         cache.LoadingCache
-	authenticator auth.Authenticator
+	authenticator *auth.Service
 	readOnlyAge   int
+	migrator      *Migrator
 }
 
 func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
@@ -35,12 +33,16 @@ func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Route
 	router.Delete("/comment/{id}", a.deleteCommentCtrl)
 	router.Put("/user/{userid}", a.setBlockCtrl)
 	router.Delete("/user/{userid}", a.deleteUserCtrl)
+	router.Get("/user/{userid}", a.getUserInfoCtrl)
 	router.Get("/deleteme", a.deleteMeRequestCtrl)
 	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Get("/export", a.exportCtrl)
 	router.Put("/pin/{id}", a.setPinCtrl)
 	router.Get("/blocked", a.blockedUsersCtrl)
 	router.Put("/readonly", a.setReadOnlyCtrl)
+	router.Put("/title/{id}", a.setTitleCtrl)
+
+	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
+
 	return router
 }
 
@@ -56,9 +58,9 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
 		return
 	}
-	a.cache.Flush(locator.SiteID, locator.URL)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope))
 	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"id": id, "locator": locator})
+	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
 
 // DELETE /user/{userid}?site=side-id - delete all user comments for requested userid
@@ -72,9 +74,25 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
 	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
+	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID})
+}
+
+// GET /user/{userid}?site=side-id - get user info for requested userid
+func (a *admin) getUserInfoCtrl(w http.ResponseWriter, r *http.Request) {
+
+	userID := chi.URLParam(r, "userid")
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[INFO] get user info for %s, site %s", userID, siteID)
+
+	ucomments, err := a.dataService.User(siteID, userID, 1, 0)
+	if err != nil || len(ucomments) == 0 {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get user info")
+		return
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, ucomments[0].User)
 }
 
 // GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
@@ -83,35 +101,57 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 
 	token := r.URL.Query().Get("token")
 
-	claims, err := a.authenticator.JWTService.Parse(token)
+	claims, err := a.authenticator.TokenService().Parse(token)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
 		return
 	}
 
-	log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID)
+	log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.Audience)
 
-	if err := a.dataService.DeleteUser(claims.SiteID, claims.User.ID); err != nil {
+	// deleteme set by deleteMeCtrl, this check just to make sure we not trying to delete with leaked token
+	if !claims.User.BoolAttr("delete_me") {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("forbidden"), "can't use provided token")
+		return
+	}
+
+	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
 		return
 	}
-	a.cache.Flush(claims.SiteID, claims.User.ID)
+
+	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
+		avatartStore := a.authenticator.AvatarProxy().Store
+		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
+			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar")
+			return
+		}
+	}
+
+	a.cache.Flush(cache.Flusher(claims.Audience).Scopes(claims.Audience, claims.User.ID, lastCommentsScope))
 	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
+	render.JSON(w, r, R.JSON{"user_id": claims.User.ID, "site_id": claims.Audience})
 }
 
-// PUT /user/{userid}?site=side-id&block=1 - block or unblock user
+// PUT /user/{userid}?site=side-id&block=1&ttl=7d - block or unblock user
 func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 	userID := chi.URLParam(r, "userid")
 	siteID := r.URL.Query().Get("site")
 	blockStatus := r.URL.Query().Get("block") == "1"
 
-	if err := a.dataService.SetBlock(siteID, userID, blockStatus); err != nil {
+	ttl := time.Duration(0) // unlimited duration by default
+	if ttlParam := r.URL.Query().Get("ttl"); ttlParam != "" {
+		if d, err := time.ParseDuration(ttlParam); err == nil {
+			ttl = d
+		}
+	}
+
+	if err := a.dataService.SetBlock(siteID, userID, blockStatus, ttl); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID, lastCommentsScope))
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
+	render.JSON(w, r, R.JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
 
 // GET /blocked?site=siteID - list blocked users
@@ -147,8 +187,25 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
 		return
 	}
-	a.cache.Flush(locator.SiteID)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
-	render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus})
+	render.JSON(w, r, R.JSON{"locator": locator, "read-only": roStatus})
+}
+
+// PUT /title/{id}?site=siteID&url=post-url - set comment PostTitle to page's title
+func (a *admin) setTitleCtrl(w http.ResponseWriter, r *http.Request) {
+	id := chi.URLParam(r, "id")
+	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
+
+	c, err := a.dataService.SetTitle(locator, id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't set title")
+		return
+	}
+	log.Printf("[INFO] set comment's title %s to %q", id, c.PostTitle)
+
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, lastCommentsScope))
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, R.JSON{"id": id, "locator": locator})
 }
 
 // PUT /verify?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
@@ -161,8 +218,8 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
-	render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus})
+	render.JSON(w, r, R.JSON{"user": userID, "verified": verifyStatus})
 }
 
 // PUT /pin/{id}?site=siteID&url=post-url&pin=1
@@ -176,33 +233,8 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
 		return
 	}
-	a.cache.Flush(locator.URL)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
-	render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus})
+	render.JSON(w, r, R.JSON{"id": commentID, "locator": locator, "pin": pinStatus})
-}
-
-// GET /export?site=site-id?mode=file|stream
-// exports all comments for siteID as json stream or gz file
-func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	var writer io.Writer = w
-	if r.URL.Query().Get("mode") == "file" {
-		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-		w.Header().Set("Content-Type", "application/gzip")
-		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
-		gzWriter := gzip.NewWriter(w)
-		defer func() {
-			if e := gzWriter.Close(); e != nil {
-				log.Printf("[WARN] can't close gzip writer, %s", e)
-			}
-		}()
-		writer = gzWriter
-	}
-
-	if _, err := a.exporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
 }
 
 func (a *admin) checkBlocked(siteID string, user store.User) bool {
@@ -215,7 +247,7 @@ func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []
 	res = make([]store.Comment, len(comments))
 
 	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin // make separate cache key for admins
+	isAdmin := err == nil && user.Admin
 
 	for i, c := range comments {
 

backend/app/rest/api/admin_test.go

@@ -1,26 +1,33 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/dgrijalva/jwt-go"
+	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/go-pkgz/auth/token"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/rest/auth"
+
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestAdmin_Delete(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -30,28 +37,120 @@ func TestAdmin_Delete(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
+	// check last comments
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments := []store.Comment{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments), "should have 2 comments")
+
+	// check multi count
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 2},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
-	body, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
+	body, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
 	assert.Equal(t, 200, code)
 	cr := store.Comment{}
 	err = json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, "", cr.Text)
 	assert.True(t, cr.Deleted)
+
+	// check last comments updated
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	comments = []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(comments), "should have 1 comments")
+
+	// check count updated
+	res, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah")
+	assert.Equal(t, 200, code)
+	b := map[string]interface{}{}
+	err = json.Unmarshal([]byte(res), &b)
+	assert.Nil(t, err)
+	t.Logf("%#v", b)
+	assert.Equal(t, 1.0, b["count"], "should report 1 comments")
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+}
+
+func TestAdmin_Title(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
+	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.String() == "/post1" {
+			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		if r.URL.String() == "/post2" {
+			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+	defer tss.Close()
+
+	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
+		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post1"}}
+	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: tss.URL + "/post2"}}
+
+	id1 := addComment(t, c1, ts)
+	addComment(t, c2, ts)
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
+	require.Equal(t, 200, code)
+	cr := store.Comment{}
+	err = json.Unmarshal([]byte(body), &cr)
+	assert.Nil(t, err)
+	assert.Equal(t, "post1 blah 123", cr.PostTitle)
 }
 
 func TestAdmin_DeleteUser(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -71,7 +170,7 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
@@ -103,9 +202,8 @@ func TestAdmin_DeleteUser(t *testing.T) {
 }
 
 func TestAdmin_Pin(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -120,7 +218,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
+		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -147,9 +245,8 @@ func TestAdmin_Pin(t *testing.T) {
 }
 
 func TestAdmin_Block(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -161,23 +258,27 @@ func TestAdmin_Block(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	assert.Nil(t, err)
 
-	block := func(val int) (code int, body []byte) {
+	block := func(val int, ttl string) (code int, body []byte) {
 		client := http.Client{}
-		req, e := http.NewRequest(http.MethodPut,
+		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
-			fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val), nil)
+		if ttl != "" {
+			url = url + "&ttl=" + ttl
+		}
+		req, e := http.NewRequest(http.MethodPut, url, nil)
 		assert.Nil(t, e)
-		req.SetBasicAuth("dev", "password")
+		req.SetBasicAuth("admin", "password")
 		resp, e := client.Do(req)
 		require.Nil(t, e)
 		body, e = ioutil.ReadAll(resp.Body)
 		assert.Nil(t, e)
-		resp.Body.Close()
+		require.Nil(t, resp.Body.Close())
 		return resp.StatusCode, body
 	}
 
-	code, body := block(1)
+	// block permanently
+	code, body := block(1, "")
 	require.Equal(t, 200, code)
-	j := JSON{}
+	j := R.JSON{}
 	err = json.Unmarshal(body, &j)
 	assert.Nil(t, err)
 	assert.Equal(t, "user1", j["user_id"])
@@ -193,17 +294,40 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Equal(t, "", comments.Comments[0].Text)
 	assert.True(t, comments.Comments[0].Deleted)
 
-	code, body = block(-1)
+	code, body = block(-1, "")
 	require.Equal(t, 200, code)
 	err = json.Unmarshal(body, &j)
 	assert.Nil(t, err)
 	assert.Equal(t, false, j["block"])
+
+	// block with ttl
+	code, _ = block(1, "50ms")
+	require.Equal(t, 200, code)
+
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, "", comments.Comments[0].Text)
+	assert.True(t, comments.Comments[0].Deleted)
+
+	srv.Cache = &cache.Nop{} // TODO: with lru cache it won't be refreshed and invalidated for long time
+	time.Sleep(50 * time.Millisecond)
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
+	assert.Equal(t, 200, code)
+	comments = commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
+	assert.Equal(t, "test test #1", comments.Comments[0].Text)
+	assert.False(t, comments.Comments[0].Deleted)
 }
 
 func TestAdmin_BlockedList(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	client := http.Client{}
 
@@ -211,32 +335,49 @@ func TestAdmin_BlockedList(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user2", 1), nil)
+		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
-	res, code := getWithAuth(t, ts.URL+"/api/v1/admin/blocked?site=radio-t")
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Equal(t, 200, code, res)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	res, err := client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
-	err = json.Unmarshal([]byte(res), &users)
+	err = json.NewDecoder(res.Body).Decode(&users)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(users), "two users blocked")
 	assert.Equal(t, "user1", users[0].ID)
 	assert.Equal(t, "user2", users[1].ID)
+
+	time.Sleep(50 * time.Millisecond)
+
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	res, err = client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, 200, res.StatusCode)
+	users = []store.BlockedUser{}
+	err = json.NewDecoder(res.Body).Decode(&users)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(users), "one user left blocked")
+
 }
 
 func TestAdmin_ReadOnly(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -258,7 +399,7 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
@@ -266,23 +407,73 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, info.ReadOnly)
 
+	// try to write comment
+	c := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+	b, err := json.Marshal(c)
+	assert.Nil(t, err, "can't marshal comment %+v", c)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
+
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	assert.Equal(t, 200, resp.StatusCode)
 	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
+
+	// try to write comment
+	c = store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
+	b, err = json.Marshal(c)
+	assert.Nil(t, err, "can't marshal comment %+v", c)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusCreated, resp.StatusCode)
+}
+
+func TestAdmin_ReadOnlyNoComments(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	client := http.Client{}
+
+	// set post to read-only
+	req, err := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
+	assert.NotNil(t, err)
+
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&format=tree")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
+	assert.True(t, comments.Info.ReadOnly)
+	t.Logf("%+v", comments)
 }
 
 func TestAdmin_ReadOnlyWithAge(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"},
@@ -300,7 +491,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
@@ -312,7 +503,7 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	assert.Equal(t, 403, resp.StatusCode)
 	require.Nil(t, err)
@@ -322,9 +513,8 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 
 }
 func TestAdmin_Verify(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -343,7 +533,7 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
 	require.Nil(t, err)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
@@ -361,7 +551,7 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
 	require.Nil(t, err)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
@@ -375,13 +565,11 @@ func TestAdmin_Verify(t *testing.T) {
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, "test test #1", comments.Comments[0].Text)
 	assert.False(t, comments.Comments[0].User.Verified)
-
 }
 
 func TestAdmin_ExportStream(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -391,17 +579,16 @@ func TestAdmin_ExportStream(t *testing.T) {
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
+	body, code := getWithAdminAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
 	assert.Equal(t, 200, code)
-	assert.Equal(t, 2, strings.Count(body, "\n"))
+	assert.Equal(t, 3, strings.Count(body, "\n"))
 	assert.Equal(t, 2, strings.Count(body, "\"text\""))
 	t.Logf("%s", body)
 }
 
 func TestAdmin_ExportFile(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -414,7 +601,7 @@ func TestAdmin_ExportFile(t *testing.T) {
 	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
 	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 
@@ -425,15 +612,14 @@ func TestAdmin_ExportFile(t *testing.T) {
 	assert.NoError(t, err)
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
+	assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
 	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
 	t.Logf("%s", string(ungzBody))
 }
 
 func TestAdmin_DeleteMeRequest(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -449,29 +635,36 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(comments), "a comment for user1")
 
-	claims := auth.CustomClaims{
+	claims := token.Claims{
-		SiteID:      "radio-t",
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
+			Audience:  "radio-t",
 			Id:        "1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
 		},
-		User: &store.User{
+		User: &token.User{
-			ID: "user1",
+			ID:      "user1",
+			Picture: "pic.image",
+			Attributes: map[string]interface{}{
+				"delete_me": true,
+			},
 		},
 	}
 
-	token, err := srv.Authenticator.JWTService.Token(&claims)
+	require.NoError(t, os.MkdirAll("/tmp/ava-remark42/42", 0700))
+	require.NoError(t, ioutil.WriteFile("/tmp/ava-remark42/42/pic.image", []byte("some image data"), 0600))
+
+	tkn, err := srv.Authenticator.TokenService().Token(claims)
 	assert.Nil(t, err)
 
 	client := http.Client{}
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	assert.Nil(t, err)
+	require.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	_, err = srv.DataService.User("radio-t", "user1", 0, 0)
@@ -479,9 +672,8 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 }
 
 func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
 		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
@@ -497,45 +689,91 @@ func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, "bad token"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 400, resp.StatusCode)
 
 	// try with bad auth
-	claims := auth.CustomClaims{
+	claims := token.Claims{
-		SiteID:      "radio-t",
 		SessionOnly: true,
 		StandardClaims: jwt.StandardClaims{
+			Audience:  "radio-t",
 			Id:        "1234567",
 			Issuer:    "remark42",
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
 		},
-		User: &store.User{
+		User: &token.User{
 			ID: "user1",
+			Attributes: map[string]interface{}{
+				"delete_me": true,
+			},
 		},
 	}
 
-	token, err := srv.Authenticator.JWTService.Token(&claims)
+	tkn, err := srv.Authenticator.TokenService().Token(claims)
 	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
+	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "bad-password")
+	req.SetBasicAuth("admin", "bad-password")
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
+	assert.Equal(t, 403, resp.StatusCode)
 
 	// try bad user
 	badClaims := claims
 	badClaims.User.ID = "no-such-id"
-	token, err = srv.Authenticator.JWTService.Token(&badClaims)
+	tkn, err = srv.Authenticator.TokenService().Token(badClaims)
 	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
+	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 400, resp.StatusCode, resp.Status)
 
+	// try without deleteme flag
+	badClaims2 := claims
+	badClaims2.User.SetBoolAttr("delete_me", false)
+	tkn, err = srv.Authenticator.TokenService().Token(badClaims2)
+	assert.Nil(t, err)
+	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.True(t, strings.Contains(string(b), "can't use provided token"))
+}
+
+func TestAdmin_GetUserInfo(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
+
+	_, err := srv.DataService.Create(c1)
+	assert.Nil(t, err)
+	_, err = srv.DataService.Create(c2)
+	assert.Nil(t, err)
+
+	body, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	assert.Equal(t, 200, code)
+	u := store.User{}
+	err = json.Unmarshal([]byte(body), &u)
+	assert.Nil(t, err)
+	assert.Equal(t, store.User{Name: "user1 name", ID: "user1", Picture: "", IP: "823688dafca7393d24c871a2da98a84d8732e927",
+		Admin: false, Blocked: false, Verified: false}, u)
+
+	_, code = get(t, fmt.Sprintf("%s/api/v1/admin/user/user1?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	assert.Equal(t, 401, code, "no auth")
+
+	_, code = getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/admin/user/userX?site=radio-t&url=https://radio-t.com/blah", ts.URL))
+	assert.Equal(t, 400, code, "no info about user")
 }

backend/app/rest/api/migrator.go

@@ -0,0 +1,236 @@
+package api
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/go-chi/chi"
+	"github.com/go-chi/render"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest"
+)
+
+// Migrator rest with import and export controllers
+type Migrator struct {
+	Cache             cache.LoadingCache
+	NativeImporter    migrator.Importer
+	DisqusImporter    migrator.Importer
+	WordPressImporter migrator.Importer
+	NativeExporter    migrator.Exporter
+	KeyStore          KeyStore
+
+	busy map[string]bool
+	lock sync.Mutex
+}
+
+// KeyStore defines sub-interface for consumers needed just a key
+type KeyStore interface {
+	Key() (key string, err error)
+}
+
+func (m *Migrator) withRoutes(router chi.Router) chi.Router {
+	router.Get("/export", m.exportCtrl)
+
+	router.Post("/import", m.importCtrl)
+	router.Post("/import/form", m.importFormCtrl)
+	router.Get("/import/wait", m.importWaitCtrl)
+	return router
+}
+
+// POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
+// imports comments from post body.
+func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
+
+	siteID := r.URL.Query().Get("site")
+
+	if m.isBusy(siteID) {
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		return
+	}
+
+	tmpfile, err := m.saveTemp(r.Body)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		return
+	}
+
+	go m.runImport(siteID, r.URL.Query().Get("provider"), tmpfile) // import runs in background and sets busy flag for site
+
+	render.Status(r, http.StatusAccepted)
+	render.JSON(w, r, R.JSON{"status": "import request accepted"})
+}
+
+// POST /import/form?secret=key&site=site-id&provider=disqus|remark|wordpress
+// imports comments from form body.
+func (m *Migrator) importFormCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
+	if m.isBusy(siteID) {
+		rest.SendErrorJSON(w, r, http.StatusConflict, errors.New("already running"), "import rejected")
+		return
+	}
+
+	if err := r.ParseMultipartForm(20 * 1024 * 1024); err != nil { // 20M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form")
+		return
+	}
+
+	file, _, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get import from the request")
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	tmpfile, err := m.saveTemp(file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save request to temp file")
+		return
+	}
+
+	go m.runImport(siteID, r.URL.Query().Get("provider"), tmpfile) // import runs in background and sets busy flag for site
+
+	render.Status(r, http.StatusAccepted)
+	render.JSON(w, r, R.JSON{"status": "import request accepted"})
+}
+
+func (m *Migrator) importWaitCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+	timeOut := time.Minute * 15
+	if v := r.URL.Query().Get("timeout"); v != "" {
+		if vv, e := time.ParseDuration(v); e == nil {
+			timeOut = vv
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeOut)
+	defer cancel()
+	for {
+		if !m.isBusy(siteID) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			render.Status(r, http.StatusGatewayTimeout)
+			render.JSON(w, r, R.JSON{"status": "timeout expired", "site_id": siteID})
+			return
+		case <-time.After(100 * time.Millisecond):
+		}
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(w, r, R.JSON{"status": "completed", "site_id": siteID})
+}
+
+// GET /export?site=site-id&secret=12345&?mode=file|stream
+// exports all comments for siteID as gz file
+func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
+
+	siteID := r.URL.Query().Get("site")
+
+	var writer io.Writer = w
+	if r.URL.Query().Get("mode") == "file" {
+		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
+		w.Header().Set("Content-Type", "application/gzip")
+		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
+		w.WriteHeader(http.StatusOK)
+		gzWriter := gzip.NewWriter(w)
+		defer func() {
+			if e := gzWriter.Close(); e != nil {
+				log.Printf("[WARN] can't close gzip writer, %s", e)
+			}
+		}()
+		writer = gzWriter
+	}
+
+	if _, err := m.NativeExporter.Export(writer, siteID); err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
+		return
+	}
+}
+
+// runImport reads from tmpfile and import for given siteID and provider
+func (m *Migrator) runImport(siteID string, provider string, tmpfile string) {
+	m.setBusy(siteID, true)
+
+	defer func() {
+		m.setBusy(siteID, false)
+		if err := os.Remove(tmpfile); err != nil {
+			log.Printf("[WARN] failed to remove tmp file %s, %v", tmpfile, err)
+		}
+	}()
+
+	var importer migrator.Importer
+	switch provider {
+	case "disqus":
+		importer = m.DisqusImporter
+	case "wordpress":
+		importer = m.WordPressImporter
+	default:
+		importer = m.NativeImporter
+	}
+	log.Printf("[DEBUG] import request for site=%s, provider=%s", siteID, provider)
+
+	fh, err := os.Open(tmpfile)
+	if err != nil {
+		log.Printf("[WARN] import failed, %v", err)
+		return
+	}
+
+	size, err := importer.Import(fh, siteID)
+	if err != nil {
+		log.Printf("[WARN] import failed, %v", err)
+		return
+	}
+	m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
+	log.Printf("[DEBUG] import request completed. site=%s, provider=%s, comments=%d", siteID, provider, size)
+}
+
+// saveTemp reads from reader and saves to temp file
+func (m *Migrator) saveTemp(r io.Reader) (string, error) {
+	tmpfile, err := ioutil.TempFile("", "remark42_import")
+	if err != nil {
+		return "", errors.Wrap(err, "can't make temp file")
+	}
+
+	if _, err = io.Copy(tmpfile, r); err != nil {
+		return "", errors.Wrap(err, "can't copy to temp file")
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		return "", errors.Wrap(err, "can't close temp file")
+	}
+
+	return tmpfile.Name(), nil
+}
+
+// isBusy checks busy flag from the map by siteID as key
+func (m *Migrator) isBusy(siteID string) bool {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	if m.busy == nil {
+		m.busy = map[string]bool{}
+	}
+	return m.busy[siteID]
+}
+
+// setBusy sets/resets busy flag to the map by siteID as key
+func (m *Migrator) setBusy(siteID string, status bool) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	if m.busy == nil {
+		m.busy = map[string]bool{}
+	}
+	m.busy[siteID] = status
+}

backend/app/rest/api/migrator_test.go

@@ -0,0 +1,468 @@
+package api
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/go-chi/chi"
+	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/auth/token"
+	"github.com/go-pkgz/rest/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/store"
+	adminstore "github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+func TestMigrator_Import(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req.SetBasicAuth("admin", "password")
+	assert.Nil(t, err)
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+
+	client = &http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+}
+
+func TestMigrator_ImportForm(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	fileWriter, err := bodyWriter.CreateFormFile("file", "import.json")
+	require.NoError(t, err)
+	_, err = io.Copy(fileWriter, r)
+	require.NoError(t, err)
+	contentType := bodyWriter.FormDataContentType()
+	require.NoError(t, bodyWriter.Close())
+
+	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
+	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err = client.Do(req)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+}
+func TestMigrator_ImportFromWP(t *testing.T) {
+	srv, ds, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
+	assert.Nil(t, err)
+	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
+
+	client = &http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	assert.NoError(t, ds.Interface.Close())
+
+	tsAccess, _, teardownAccess := startupT(t)
+	defer teardownAccess()
+
+	res, code := get(t, tsAccess.URL+"/api/v1/last/10?site=radio-t")
+	require.Equal(t, 200, code)
+	comments := []store.Comment{}
+	err = json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(comments), "should have 3 comments")
+	t.Logf("%+v", comments)
+	assert.Equal(t, "<p>Looks like <a href=\"http://releases.rancher.com/os/latest\" rel=\"nofollow\">http://releases.rancher.com/os/latest</a> is no longer hosted - installs using this <code>base-url</code> are failing.</p>\n\n<p>I switched to Github with success:</p>\n\n<pre><code>set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1\n</code></pre>\n\n<p>Thanks for the article!</p>\n",
+		comments[0].Text)
+}
+
+func TestMigrator_ImportRejected(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
+	assert.Nil(t, err)
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func TestMigrator_ImportDouble(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	recs := []string{}
+	for i := 0; i < 5000; i++ {
+		recs = append(recs, fmt.Sprintf(tmpl, i))
+	}
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req.SetBasicAuth("admin", "password")
+	assert.Nil(t, err)
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	client = &http.Client{Timeout: 1 * time.Second}
+	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req.SetBasicAuth("admin", "password")
+	assert.Nil(t, err)
+	resp, err = client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusConflict, resp.StatusCode)
+
+}
+
+func TestMigrator_ImportWaitExpired(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
+	recs := []string{}
+	for i := 0; i < 5000; i++ {
+		recs = append(recs, fmt.Sprintf(tmpl, i))
+	}
+	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req.SetBasicAuth("admin", "password")
+	require.Nil(t, err)
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
+
+	client = &http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+}
+
+func TestMigrator_Export(t *testing.T) {
+	srv, _, ts := prepImportSrv(t)
+	assert.NotNil(t, srv)
+	defer cleanupImportSrv(srv, ts)
+
+	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
+	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
+
+	// import comments first
+	client := &http.Client{Timeout: 1 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, http.StatusAccepted, resp.StatusCode)
+	client = &http.Client{Timeout: 10 * time.Second}
+	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req.SetBasicAuth("admin", "password")
+	assert.NoError(t, err)
+	resp, err = client.Do(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	// check file mode
+	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, 200, resp.StatusCode)
+	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
+
+	ungzReader, err := gzip.NewReader(resp.Body)
+	assert.NoError(t, err)
+	ungzBody, err := ioutil.ReadAll(ungzReader)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, strings.Count(string(ungzBody), "\n"))
+	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
+	t.Logf("%s", string(ungzBody))
+
+	// check stream mode
+	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, 200, resp.StatusCode)
+	require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, strings.Count(string(body), "\n"))
+	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
+	t.Logf("%s", string(body))
+
+	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
+	require.Nil(t, err)
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+}
+
+func prepImportSrv(t *testing.T) (svc *Migrator, ds *service.DataStore, ts *httptest.Server) {
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.Nil(t, err)
+	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
+	svc = &Migrator{
+		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
+		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
+		NativeImporter:    &migrator.Native{DataStore: dataStore},
+		NativeExporter:    &migrator.Native{DataStore: dataStore},
+		Cache:             &cache.Nop{},
+		KeyStore:          adminStore,
+	}
+
+	a := auth.NewService(auth.Opts{
+		AdminPasswd:  "password",
+		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
+		Issuer:       "test",
+	})
+
+	am := a.Middleware()
+	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
+	ts = httptest.NewServer(routes)
+	return svc, dataStore, ts
+}
+
+func cleanupImportSrv(_ *Migrator, ts *httptest.Server) {
+	ts.Close()
+	_ = os.Remove(testDb)
+}
+
+var xmlTestWP = `
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0"
+	xmlns:excerpt="http://wordpress.org/export/1.2/excerpt/"
+	xmlns:content="http://purl.org/rss/1.0/modules/content/"
+	xmlns:wfw="http://wellformedweb.org/CommentAPI/"
+	xmlns:dc="http://purl.org/dc/elements/1.1/"
+	xmlns:wp="http://wordpress.org/export/1.2/"
+>
+
+<channel>
+	<title>Real Men Wear Dress.es</title>
+	<link>https://realmenweardress.es</link>
+	<description>SuperAdmin&#039;s gaming and technological musings</description>
+	<pubDate>Mon, 23 Jul 2018 10:21:47 +0000</pubDate>
+	<language>en-US</language>
+	<wp:wxr_version>1.2</wp:wxr_version>
+	<wp:base_site_url>https://realmenweardress.es</wp:base_site_url>
+	<wp:base_blog_url>https://realmenweardress.es</wp:base_blog_url>
+
+	<wp:author><wp:author_id>2</wp:author_id><wp:author_login><![CDATA[SuperAdmin]]></wp:author_login><wp:author_email><![CDATA[superadmin@super.eu]]></wp:author_email><wp:author_display_name><![CDATA[SuperAdmin]]></wp:author_display_name><wp:author_first_name><![CDATA[SuperAdmin]]></wp:author_first_name><wp:author_last_name><![CDATA[superadmin]]></wp:author_last_name></wp:author>
+	<wp:author><wp:author_id>1</wp:author_id><wp:author_login><![CDATA[admin]]></wp:author_login><wp:author_email><![CDATA[superadmin@superadmin.co.uk]]></wp:author_email><wp:author_display_name><![CDATA[admin]]></wp:author_display_name><wp:author_first_name><![CDATA[]]></wp:author_first_name><wp:author_last_name><![CDATA[]]></wp:author_last_name></wp:author>
+
+	<wp:category>
+		<wp:term_id>25</wp:term_id>
+		<wp:category_nicename><![CDATA[cataclysm]]></wp:category_nicename>
+		<wp:category_parent><![CDATA[]]></wp:category_parent>
+		<wp:cat_name><![CDATA[Cataclysm]]></wp:cat_name>
+	</wp:category>
+
+	<wp:tag>
+		<wp:term_id>39</wp:term_id>
+		<wp:tag_slug><![CDATA[addons]]></wp:tag_slug>
+		<wp:tag_name><![CDATA[addons]]></wp:tag_name>
+	</wp:tag>
+
+	<generator>https://wordpress.org/?v=4.8.1</generator>
+
+	<item>
+		<title>Post without comments</title>
+		<link>https://realmenweardress.es/2010/06/hello-world/screenshot_013110_200413/</link>
+		<pubDate>Sat, 19 Jun 2010 08:34:13 +0000</pubDate>
+		<dc:creator><![CDATA[admin]]></dc:creator>
+		<guid isPermaLink="false">http://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413.jpeg</guid>
+		<description></description>
+		<content:encoded><![CDATA[So you can actually fly into the well it appears and if your lucky you stay mounted. I imagine it terrifies the poor rats.]]></content:encoded>
+		<excerpt:encoded><![CDATA[]]></excerpt:encoded>
+		<wp:post_id>6</wp:post_id>
+		<wp:post_date><![CDATA[2010-06-19 08:34:13]]></wp:post_date>
+		<wp:post_date_gmt><![CDATA[2010-06-19 08:34:13]]></wp:post_date_gmt>
+		<wp:comment_status><![CDATA[open]]></wp:comment_status>
+		<wp:ping_status><![CDATA[open]]></wp:ping_status>
+		<wp:post_name><![CDATA[screenshot_013110_200413]]></wp:post_name>
+		<wp:status><![CDATA[inherit]]></wp:status>
+		<wp:post_parent>1</wp:post_parent>
+		<wp:menu_order>0</wp:menu_order>
+		<wp:post_type><![CDATA[attachment]]></wp:post_type>
+		<wp:post_password><![CDATA[]]></wp:post_password>
+		<wp:is_sticky>0</wp:is_sticky>
+		<wp:attachment_url><![CDATA[https://realmenweardress.es/wp-content/uploads/2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:attachment_url>
+		<wp:postmeta>
+			<wp:meta_key><![CDATA[_wp_attached_file]]></wp:meta_key>
+			<wp:meta_value><![CDATA[2010/06/ScreenShot_013110_200413-e1277214413194.jpeg]]></wp:meta_value>
+		</wp:postmeta>
+	</item>
+	<item>
+		<title>Post with comments. One is not approved</title>
+		<link>https://realmenweardress.es/2010/07/do-you-rp/</link>
+		<pubDate>Mon, 19 Jul 2010 14:24:22 +0000</pubDate>
+		<dc:creator><![CDATA[SuperAdmin]]></dc:creator>
+		<guid isPermaLink="false">http://realmenweardress.es/?p=100</guid>
+		<description></description>
+		<content:encoded><![CDATA[<a href="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546180886.jpeg"><img class="size-thumbnail wp-image-102 alignleft" title="I need to stand on things else I can't reach" src="http://realmenweardress.es/wp-content/uploads/2010/07/ScreenShot_071410_230307-e1279546270587-120x120.jpg" alt="I need to stand on things else I can't reach" width="120" height="120" /></a>Meet Grokknomel?]]></content:encoded>
+		<excerpt:encoded><![CDATA[]]></excerpt:encoded>
+		<wp:post_id>100</wp:post_id>
+		<wp:post_date><![CDATA[2010-07-19 14:24:22]]></wp:post_date>
+		<wp:post_date_gmt><![CDATA[2010-07-19 14:24:22]]></wp:post_date_gmt>
+		<wp:comment_status><![CDATA[open]]></wp:comment_status>
+		<wp:ping_status><![CDATA[open]]></wp:ping_status>
+		<wp:post_name><![CDATA[do-you-rp]]></wp:post_name>
+		<wp:status><![CDATA[publish]]></wp:status>
+		<wp:post_parent>0</wp:post_parent>
+		<wp:menu_order>0</wp:menu_order>
+		<wp:post_type><![CDATA[post]]></wp:post_type>
+		<wp:post_password><![CDATA[]]></wp:post_password>
+		<wp:is_sticky>0</wp:is_sticky>
+		<category domain="post_tag" nicename="alts"><![CDATA[alts]]></category>
+		<category domain="post_tag" nicename="role-playing"><![CDATA[role playing]]></category>
+		<category domain="category" nicename="stuff"><![CDATA[Stuff]]></category>
+		<category domain="post_tag" nicename="wierd-in-a-cant-quite-help-myself-way"><![CDATA[wierd in a can't quite help myself way]]></category>
+		<wp:postmeta>
+			<wp:meta_key><![CDATA[_edit_last]]></wp:meta_key>
+			<wp:meta_value><![CDATA[2]]></wp:meta_value>
+		</wp:postmeta>
+		<wp:comment>
+			<wp:comment_id>8</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser1]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[superuser1@aol.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://superuser1.blogspot.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[79.141.141.73]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-20 12:08:08]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-20 12:08:08]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[I do catch myself]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>9</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser2]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[superuser2@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://thewowstorm.wordpress.com</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[97.36.113.1]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-20 13:09:25]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-20 13:09:25]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[I think it us inherent in the game to start seeing your character as a personality]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[0]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>13</wp:comment_id>
+			<wp:comment_author><![CDATA[Wednesday Reading &laquo; Cynwise&#039;s Battlefield Manual]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[]]></wp:comment_author_email>
+			<wp:comment_author_url>http://cynwise.wordpress.com/2010/07/21/wednesday-reading-8/</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[74.200.244.101]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-07-21 14:02:08]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-07-21 14:02:08]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[[...] I know I&#8217;m a bit loony with my attachment to my bankers.  I&#8217;m glad I&#8217;m not the only one. [...]]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[pingback]]></wp:comment_type>
+			<wp:comment_parent>0</wp:comment_parent>
+			<wp:comment_user_id>0</wp:comment_user_id>
+		</wp:comment>
+		<wp:comment>
+			<wp:comment_id>14</wp:comment_id>
+			<wp:comment_author><![CDATA[SuperUser3]]></wp:comment_author>
+			<wp:comment_author_email><![CDATA[blablah@gmail.com]]></wp:comment_author_email>
+			<wp:comment_author_url>http://realmenweardress.es</wp:comment_author_url>
+			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
+			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
+			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
+			<wp:comment_content><![CDATA[Looks like http://releases.rancher.com/os/latest is no longer hosted - installs using this 'base-url' are failing.
+
+I switched to Github with success:
+
+'''
+set base-url https://github.com/rancher/os/releases/download/v1.1.1-rc1
+'''
+
+Thanks for the article!]]></wp:comment_content>
+			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
+			<wp:comment_type><![CDATA[]]></wp:comment_type>
+			<wp:comment_parent>13</wp:comment_parent>
+			<wp:comment_user_id>2</wp:comment_user_id>
+		</wp:comment>
+	</item>
+	</channel>
+</rss>
+`

backend/app/rest/api/rest.go

@@ -0,0 +1,348 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/didip/tollbooth"
+	"github.com/didip/tollbooth_chi"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/cors"
+	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
+	"github.com/go-pkgz/rest/logger"
+	"github.com/pkg/errors"
+	"github.com/rakyll/statik/fs"
+
+	"github.com/umputun/remark/backend/app/notify"
+	"github.com/umputun/remark/backend/app/rest"
+	"github.com/umputun/remark/backend/app/rest/proxy"
+	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+// Rest is a rest access server
+type Rest struct {
+	Version string
+
+	DataService      *service.DataStore
+	Authenticator    *auth.Service
+	Cache            cache.LoadingCache
+	ImageProxy       *proxy.Image
+	CommentFormatter *store.CommentFormatter
+	Migrator         *Migrator
+	NotifyService    *notify.Service
+
+	WebRoot         string
+	RemarkURL       string
+	ReadOnlyAge     int
+	SharedSecret    string
+	ScoreThresholds struct {
+		Low      int
+		Critical int
+	}
+	UpdateLimiter float64
+
+	SSLConfig   SSLConfig
+	httpsServer *http.Server
+	httpServer  *http.Server
+	lock        sync.Mutex
+
+	adminService admin
+}
+
+const hardBodyLimit = 1024 * 64 // limit size of body
+
+const lastCommentsScope = "last"
+
+type commentsWithInfo struct {
+	Comments []store.Comment `json:"comments"`
+	Info     store.PostInfo  `json:"info,omitempty"`
+}
+
+// Run the lister and request's router, activate rest server
+func (s *Rest) Run(port int) {
+	switch s.SSLConfig.SSLMode {
+	case None:
+		log.Printf("[INFO] activate http rest server on port %d", port)
+
+		s.lock.Lock()
+		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.lock.Unlock()
+
+		err := s.httpServer.ListenAndServe()
+		log.Printf("[WARN] http server terminated, %s", err)
+	case Static:
+		log.Printf("[INFO] activate https server in 'static' mode on port %d", s.SSLConfig.Port)
+
+		s.lock.Lock()
+		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.lock.Unlock()
+
+		go func() {
+			log.Printf("[INFO] activate http redirect server on port %d", port)
+			err := s.httpServer.ListenAndServe()
+			log.Printf("[WARN] http redirect server terminated, %s", err)
+		}()
+
+		err := s.httpsServer.ListenAndServeTLS(s.SSLConfig.Cert, s.SSLConfig.Key)
+		log.Printf("[WARN] https server terminated, %s", err)
+	case Auto:
+		log.Printf("[INFO] activate https server in 'auto' mode on port %d", s.SSLConfig.Port)
+
+		m := s.makeAutocertManager()
+		s.lock.Lock()
+		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.lock.Unlock()
+
+		go func() {
+			log.Printf("[INFO] activate http challenge server on port %d", port)
+
+			err := s.httpServer.ListenAndServe()
+			log.Printf("[WARN] http challenge server terminated, %s", err)
+		}()
+
+		err := s.httpsServer.ListenAndServeTLS("", "")
+		log.Printf("[WARN] https server terminated, %s", err)
+	}
+}
+
+// Shutdown rest http server
+func (s *Rest) Shutdown() {
+	log.Print("[WARN] shutdown rest server")
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	s.lock.Lock()
+	if s.httpServer != nil {
+		if err := s.httpServer.Shutdown(ctx); err != nil {
+			log.Printf("[DEBUG] http shutdown error, %s", err)
+		}
+		log.Print("[DEBUG] shutdown http server completed")
+	}
+
+	if s.httpsServer != nil {
+		log.Print("[WARN] shutdown https server")
+		if err := s.httpsServer.Shutdown(ctx); err != nil {
+			log.Printf("[DEBUG] https shutdown error, %s", err)
+		}
+		log.Print("[DEBUG] shutdown https server completed")
+	}
+	s.lock.Unlock()
+}
+
+func (s *Rest) makeHTTPServer(port int, router http.Handler) *http.Server {
+	return &http.Server{
+		Addr:              fmt.Sprintf(":%d", port),
+		Handler:           router,
+		ReadHeaderTimeout: 5 * time.Second,
+		WriteTimeout:      120 * time.Second, // TODO: such a long timeout needed for blocking export (backup) request
+		IdleTimeout:       30 * time.Second,
+	}
+}
+
+func (s *Rest) routes() chi.Router {
+	router := chi.NewRouter()
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+	router.Use(R.AppInfo("remark42", "umputun", s.Version), R.Ping)
+
+	s.adminService = admin{
+		dataService:   s.DataService,
+		migrator:      s.Migrator,
+		cache:         s.Cache,
+		authenticator: s.Authenticator,
+		readOnlyAge:   s.ReadOnlyAge,
+	}
+
+	corsMiddleware := cors.New(cors.Options{
+		AllowedOrigins:   []string{"*"},
+		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
+		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-XSRF-Token", "X-JWT"},
+		ExposedHeaders:   []string{"Authorization"},
+		AllowCredentials: true,
+		MaxAge:           300,
+	})
+	router.Use(corsMiddleware.Handler)
+
+	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+
+	authHandler, avatarHandler := s.Authenticator.Handlers()
+
+	router.Group(func(r chi.Router) {
+		l := logger.New(logger.Flags(logger.All), logger.Log(log.Default()), logger.IPfn(ipFn), logger.Prefix("[INFO]"))
+		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
+		r.Mount("/auth", authHandler)
+	})
+
+	router.Group(func(r chi.Router) {
+		r.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+		r.Mount("/avatar", avatarHandler)
+	})
+
+	authMiddleware := s.Authenticator.Middleware()
+
+	// api routes
+	router.Route("/api/v1", func(rapi chi.Router) {
+
+		rapi.Group(func(rava chi.Router) {
+			rava.Use(logger.New(logger.Flags(logger.None)).Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)))
+			rava.Mount("/avatar", avatarHandler)
+		})
+
+		// open routes
+		rapi.Group(func(ropen chi.Router) {
+			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			ropen.Use(authMiddleware.Trace)
+			ropen.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Get("/find", s.findCommentsCtrl)
+			ropen.Get("/id/{id}", s.commentByIDCtrl)
+			ropen.Get("/comments", s.findUserCommentsCtrl)
+			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
+			ropen.Get("/count", s.countCtrl)
+			ropen.Post("/counts", s.countMultiCtrl)
+			ropen.Get("/list", s.listCtrl)
+			ropen.Get("/config", s.configCtrl)
+			ropen.Post("/preview", s.previewCommentCtrl)
+			ropen.Get("/info", s.infoCtrl)
+
+			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Mount("/img", s.ImageProxy.Routes())
+		})
+
+		// protected routes, require auth
+		rapi.Group(func(rauth chi.Router) {
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			rauth.Get("/user", s.userInfoCtrl)
+			rauth.Get("/userdata", s.userAllDataCtrl)
+
+			// admin routes, admin users only
+			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		})
+
+		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Flags(logger.All), logger.Log(log.Default()),
+				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+
+			rauth.Put("/comment/{id}", s.updateCommentCtrl)
+			rauth.Post("/comment", s.createCommentCtrl)
+			rauth.Put("/vote/{id}", s.voteCtrl)
+			rauth.Post("/deleteme", s.deleteMeCtrl)
+		})
+	})
+
+	// respond to /robots.txt with the list of allowed paths
+	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
+		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+			for i := range allowed {
+				allowed[i] = "Allow: /api/v1" + allowed[i]
+			}
+			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+		})
+
+	// respond to /index.html with the content of getstarted.html under /web root
+	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
+		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
+			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
+			if err != nil {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			render.HTML(w, r, string(data))
+		})
+
+	// file server for static content from /web
+	addFileServer(router, "/web", http.Dir(s.WebRoot))
+	return router
+}
+
+// serves static files from /web or embedded by statik
+func addFileServer(r chi.Router, path string, root http.FileSystem) {
+
+	var webFS http.Handler
+
+	statikFS, err := fs.New()
+	if err == nil {
+		log.Printf("[INFO] run file server for %s, embedded", root)
+		webFS = http.FileServer(statikFS)
+	}
+	if err != nil {
+		log.Printf("[DEBUG] no embedded assets loaded, %s", err)
+		log.Printf("[INFO] run file server for %s, path %s", root, path)
+		webFS = http.FileServer(root)
+	}
+
+	origPath := path
+	webFS = http.StripPrefix(path, webFS)
+	if path != "/" && path[len(path)-1] != '/' {
+		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
+		path += "/"
+	}
+	path += "*"
+
+	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
+		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// don't show dirs, just serve files
+			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
+				http.NotFound(w, r)
+				return
+			}
+			webFS.ServeHTTP(w, r)
+		}))
+}
+
+func encodeJSONWithHTML(v interface{}) ([]byte, error) {
+	buf := &bytes.Buffer{}
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(v); err != nil {
+		return nil, errors.Wrap(err, "json encoding failed")
+	}
+	return buf.Bytes(), nil
+}
+
+func filterComments(comments []store.Comment, fn func(c store.Comment) bool) []store.Comment {
+	filtered := []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			filtered = append(filtered, c)
+		}
+	}
+	return filtered
+}
+
+// URLKey gets url from request to use it as cache key
+// admins will have different keys in order to prevent leak of admin-only data to regular users
+func URLKey(r *http.Request) string {
+	adminPrefix := "admin!!"
+	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
+	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
+		key = adminPrefix + key
+	}
+	return key
+}

backend/app/rest/api/rest_private.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"strings"
 	"time"
@@ -13,13 +12,15 @@ import (
 	jwt "github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/go-pkgz/auth/token"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 	multierror "github.com/hashicorp/go-multierror"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
 
-	"github.com/umputun/remark/app/rest"
+	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
+	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
-	"github.com/umputun/remark/app/store/service"
 )
 
 // POST /comment - adds comment, resets all immutable fields
@@ -31,38 +32,35 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	user, err := rest.GetUserInfo(r)
+	user := rest.MustGetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	log.Printf("[DEBUG] create comment %+v", comment)
 
 	comment.PrepareUntrusted() // clean all fields user not supposed to set
 	comment.User = user
 	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
 
 	comment.Orig = comment.Text // original comment text, prior to md render
-	if err = s.DataService.ValidateComment(&comment); err != nil {
+	if err := s.DataService.ValidateComment(&comment); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
 		return
 	}
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
+	comment = s.CommentFormatter.Format(comment)
-	comment.Text = s.ImageProxy.Convert(comment.Text)
+
 	// check if user blocked
 	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
 		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
 		return
 	}
 
-	if s.ReadOnlyAge > 0 {
+	if s.isReadOnly(comment.Locator) {
-		if info, e := s.DataService.Info(comment.Locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		return
-			return
-		}
 	}
 
 	id, err := s.DataService.Create(comment)
+	if err == service.ErrRestrictedWordsFound {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		return
+	}
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
 		return
@@ -74,7 +72,14 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
 		return
 	}
-	s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)
+	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+		Scopes(comment.Locator.URL, lastCommentsScope, comment.User.ID, comment.Locator.SiteID))
+
+	if s.NotifyService != nil {
+		s.NotifyService.Submit(finalComment)
+	}
+
+	log.Printf("[DEBUG] created commend %+v", finalComment)
 
 	render.Status(r, http.StatusCreated)
 	render.JSON(w, r, &finalComment)
@@ -86,6 +91,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	edit := struct {
 		Text    string
 		Summary string
+		Delete  bool
 	}{}
 
 	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
@@ -93,17 +99,14 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	user, err := rest.GetUserInfo(r)
+	user := rest.MustGetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
 
 	log.Printf("[DEBUG] update comment %s", id)
 
 	var currComment store.Comment
+	var err error
 	if currComment, err = s.DataService.Get(locator, id); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
 		return
@@ -114,32 +117,30 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
-	text = s.ImageProxy.Convert(text)
 	editReq := service.EditRequest{
-		Text:    text,
+		Text:    s.CommentFormatter.FormatText(edit.Text),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
+		Delete:  edit.Delete,
 	}
 
 	res, err := s.DataService.EditComment(locator, id, editReq)
+	if err == service.ErrRestrictedWordsFound {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
+		return
+	}
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
 		return
 	}
 
-	s.Cache.Flush(locator.URL, "last", user.ID)
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.SiteID, locator.URL, lastCommentsScope, user.ID))
 	render.JSON(w, r, res)
 }
 
 // GET /user?site=siteID - returns user info
 func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
+	user := rest.MustGetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-
 	if siteID := r.URL.Query().Get("site"); siteID != "" {
 		user.Verified = s.DataService.IsVerified(siteID, user.ID)
 	}
@@ -149,35 +150,37 @@ func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
 
 // PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
 func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
-
+	user := rest.MustGetUserInfo(r)
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	id := chi.URLParam(r, "id")
 	log.Printf("[DEBUG] vote for comment %s", id)
 
 	vote := r.URL.Query().Get("vote") == "1"
 
+	if s.isReadOnly(locator) {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
+		return
+	}
+
+	// check if user blocked
+	if s.adminService.checkBlocked(locator.SiteID, user) {
+		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
+		return
+	}
+
 	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
 		return
 	}
-	s.Cache.Flush(locator.URL)
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
-	render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
+	render.JSON(w, r, R.JSON{"id": comment.ID, "score": comment.Score})
 }
 
 // GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
 func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
-	user, err := rest.GetUserInfo(r)
+	user := rest.MustGetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
 	userB, err := json.Marshal(&user)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
@@ -200,7 +203,6 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var merr error
-
 	merr = multierror.Append(merr, write([]byte(`{"info": `)))     // send user prefix
 	merr = multierror.Append(merr, write(userB))                   // send user info
 	merr = multierror.Append(merr, write([]byte(`, "comments":`))) // send comments prefix
@@ -235,28 +237,41 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 // POST /deleteme?site_id=site - requesting delete of all user info
 // makes jwt with user info and sends it back as a part of json response
 func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
+	user := rest.MustGetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
 	siteID := r.URL.Query().Get("site")
 
-	claims := auth.CustomClaims{
+	claims := token.Claims{
-		SiteID: siteID,
 		StandardClaims: jwt.StandardClaims{
+			Audience:  siteID,
 			Issuer:    "remark42",
 			ExpiresAt: time.Now().AddDate(0, 3, 0).Unix(),
 			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
 		},
-		User: &user,
+		User: &token.User{
+			ID:   user.ID,
+			Name: user.Name,
+			Attributes: map[string]interface{}{
+				"delete_me": true, // prevents this token from being used for login
+			},
+		},
 	}
 
-	tokenStr, err := s.Authenticator.JWTService.Token(&claims)
+	tokenStr, err := s.Authenticator.TokenService().Token(claims)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
 		return
 	}
-	link := fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", s.RemarkURL, tokenStr)
+
-	render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
+	link := fmt.Sprintf("%s/web/deleteme.html?token=%s", s.RemarkURL, tokenStr)
+	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
+}
+
+func (s *Rest) isReadOnly(locator store.Locator) bool {
+	if s.ReadOnlyAge > 0 {
+		// check RO by age
+		if info, e := s.DataService.Info(locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
+			return true
+		}
+	}
+	return s.DataService.IsReadOnly(locator) // ro manually
 }

backend/app/rest/api/rest_private_test.go

@@ -10,25 +10,26 @@ import (
 	"testing"
 	"time"
 
+	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestRest_Create(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	resp, err := post(t, ts.URL+"/api/v1/comment",
 		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
 	assert.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
-	c := JSON{}
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	t.Log(string(b))
+	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
 	assert.Nil(t, err)
 	loc := c["locator"].(map[string]interface{})
@@ -38,9 +39,8 @@ func TestRest_Create(t *testing.T) {
 }
 
 func TestRest_CreateOldPost(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	// make old, but not too old comment
 	old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
@@ -72,9 +72,8 @@ func TestRest_CreateOldPost(t *testing.T) {
 }
 
 func TestRest_CreateTooBig(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
 
@@ -83,7 +82,7 @@ func TestRest_CreateTooBig(t *testing.T) {
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
-	c := JSON{}
+	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
 	assert.Nil(t, err)
 	assert.Equal(t, "comment text exceeded max allowed size 4000 (4001)", c["error"])
@@ -95,18 +94,35 @@ func TestRest_CreateTooBig(t *testing.T) {
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	b, err = ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
-	c = JSON{}
+	c = R.JSON{}
 	err = json.Unmarshal(b, &c)
 	assert.Nil(t, err)
 	assert.Equal(t, "http: request body too large", c["error"])
 	assert.Equal(t, "can't bind comment", c["details"])
 }
 
+func TestRest_CreateWithRestrictedWord(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	badComment := fmt.Sprintf(`{"text": "What the duck is that?", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", badComment)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	c := R.JSON{}
+	err = json.Unmarshal(b, &c)
+	assert.Nil(t, err)
+	assert.Equal(t, "comment contains restricted words", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_CreateRejected(t *testing.T) {
 
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 	body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`
 
 	// try to create without auth
@@ -114,42 +130,49 @@ func TestRest_CreateRejected(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
 func TestRest_CreateAndGet(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	// create comment
 	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "**test** *123* http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
+		`{"text": "**test** *123*\n\n http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
 	require.Nil(t, err)
 	require.Equal(t, http.StatusCreated, resp.StatusCode)
 	b, err := ioutil.ReadAll(resp.Body)
 	assert.Nil(t, err)
-	c := JSON{}
+	c := R.JSON{}
 	err = json.Unmarshal(b, &c)
 	assert.Nil(t, err)
 
 	id := c["id"].(string)
 
-	// get created comment by id
+	// get created comment by id as admin
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	comment := store.Comment{}
 	err = json.Unmarshal([]byte(res), &comment)
 	assert.Nil(t, err)
-	assert.Equal(t, `<p><strong>test</strong> <em>123</em> <a href="http://radio-t.com" rel="nofollow">http://radio-t.com</a></p>`+"\n", comment.Text)
+	assert.Equal(t, "<p><strong>test</strong> <em>123</em></p>\n\n<p><a href=\"http://radio-t.com\" rel=\"nofollow\">http://radio-t.com</a></p>\n", comment.Text)
-	assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig)
+	assert.Equal(t, "**test** *123*\n\n http://radio-t.com", comment.Orig)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false,
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
+		IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
 		comment.User)
 	t.Logf("%+v", comment)
+
+	// get created comment by id as non-admin
+	res, code = getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, 200, code)
+	comment = store.Comment{}
+	err = json.Unmarshal([]byte(res), &comment)
+	assert.Nil(t, err)
+	assert.Equal(t, store.User{Name: "admin", ID: "admin", Admin: true, Blocked: false, IP: ""}, comment.User, "no ip")
 }
 
 func TestRest_Update(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -159,7 +182,7 @@ func TestRest_Update(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
 		strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
 	assert.Nil(t, err)
 	body, err := ioutil.ReadAll(b.Body)
@@ -177,7 +200,7 @@ func TestRest_Update(t *testing.T) {
 	assert.True(t, time.Since(c2.Edit.Timestamp) < 1*time.Second)
 
 	// read updated comment
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	res, code := getWithAdminAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
 	assert.Equal(t, 200, code)
 	c3 := store.Comment{}
 	err = json.Unmarshal([]byte(res), &c3)
@@ -185,10 +208,71 @@ func TestRest_Update(t *testing.T) {
 	assert.Equal(t, c2, c3, "same as response from update")
 }
 
+func TestRest_UpdateDelete(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id := addComment(t, c1, ts)
+
+	// check multi count updated
+	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	j := []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	assert.Nil(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 1},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+
+	// delete a comment
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"delete": true, "summary":"removed by user"}`))
+	require.NoError(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err := client.Do(req)
+	require.NoError(t, err)
+	body, err := ioutil.ReadAll(b.Body)
+	require.NoError(t, err)
+	assert.Equal(t, 200, b.StatusCode, string(body))
+
+	// comments returned by update
+	c2 := store.Comment{}
+	err = json.Unmarshal(body, &c2)
+	require.NoError(t, err)
+	assert.Equal(t, id, c2.ID)
+	assert.True(t, c2.Deleted)
+
+	// read updated comment
+	res, code := getWithDevAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
+	assert.Equal(t, 200, code)
+	c3 := store.Comment{}
+	err = json.Unmarshal([]byte(res), &c3)
+	assert.Nil(t, err)
+	assert.Equal(t, "", c3.Text)
+	assert.Equal(t, "", c3.Orig)
+	assert.True(t, c3.Deleted)
+
+	// check multi count updated
+	resp, err = post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	bb, err = ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	j = []store.PostInfo{}
+	err = json.Unmarshal(bb, &j)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 0},
+		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
+}
+
 func TestRest_UpdateNotOwner(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
@@ -199,7 +283,7 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
 		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	b, err := client.Do(req)
 	assert.Nil(t, err)
 	body, err := ioutil.ReadAll(b.Body)
@@ -211,16 +295,40 @@ func TestRest_UpdateNotOwner(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
 		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	b, err = client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
 }
 
+func TestRest_UpdateWithRestrictedWords(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "What the quack is that?", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	id := addComment(t, c1, ts)
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
+		strings.NewReader(`{"text":"What the duck is that?", "summary":"my edit"}`))
+	assert.Nil(t, err)
+	req.Header.Add("X-JWT", devToken)
+	b, err := client.Do(req)
+	assert.Nil(t, err)
+	body, err := ioutil.ReadAll(b.Body)
+	assert.Nil(t, err)
+	c := R.JSON{}
+	err = json.Unmarshal(body, &c)
+	assert.Nil(t, err)
+	assert.Equal(t, 400, b.StatusCode, string(body))
+	assert.Equal(t, "comment contains restricted words", c["error"])
+	assert.Equal(t, "invalid comment", c["details"])
+}
+
 func TestRest_Vote(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
@@ -235,7 +343,7 @@ func TestRest_Vote(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
+		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
 		return resp.StatusCode
@@ -249,7 +357,7 @@ func TestRest_Vote(t *testing.T) {
 	err := json.Unmarshal([]byte(body), &cr)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"dev": true}, cr.Votes)
+	assert.Equal(t, map[string]bool{"admin": true}, cr.Votes)
 
 	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
 	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
@@ -262,9 +370,8 @@ func TestRest_Vote(t *testing.T) {
 }
 
 func TestRest_UserAllData(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	// write 3 comments
 	user := store.User{ID: "dev", Name: "user name 1"}
@@ -284,7 +391,7 @@ func TestRest_UserAllData(t *testing.T) {
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, 200, resp.StatusCode)
@@ -295,7 +402,7 @@ func TestRest_UserAllData(t *testing.T) {
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
 	assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
 	t.Logf("%s", string(ungzBody))
 
@@ -306,7 +413,8 @@ func TestRest_UserAllData(t *testing.T) {
 
 	err = json.Unmarshal(ungzBody, &parsed)
 	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true}, parsed.Info)
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+		Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, parsed.Info)
 	assert.Equal(t, 3, len(parsed.Comments))
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
@@ -317,9 +425,8 @@ func TestRest_UserAllData(t *testing.T) {
 }
 
 func TestRest_UserAllDataManyComments(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	user := store.User{ID: "dev", Name: "user name 1"}
 	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
@@ -335,7 +442,7 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	client := &http.Client{Timeout: 1 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
 	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
 	require.Nil(t, err)
 	require.Equal(t, 200, resp.StatusCode)
@@ -346,19 +453,18 @@ func TestRest_UserAllDataManyComments(t *testing.T) {
 	ungzBody, err := ioutil.ReadAll(ungzReader)
 	assert.NoError(t, err)
 	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
+		`{"info": {"name":"developer one","id":"dev","picture":"http://example.com/pic.png","ip":"127.0.0.1","admin":false}, "comments":[{`))
 	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
 }
 
 func TestRest_DeleteMe(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
@@ -372,10 +478,10 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Equal(t, "dev", m["user_id"])
 
 	token := m["token"]
-	claims, err := srv.Authenticator.JWTService.Parse(token)
+	claims, err := srv.Authenticator.TokenService().Parse(token)
 	assert.Nil(t, err)
 	assert.Equal(t, "dev", claims.User.ID)
-	assert.Equal(t, "https://demo.remark42.com/api/v1/admin/deleteme?token="+token, m["link"])
+	assert.Equal(t, "https://demo.remark42.com/web/deleteme.html?token="+token, m["link"])
 
 	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
 	assert.Nil(t, err)

backend/app/rest/api/rest_public.go

@@ -3,18 +3,18 @@ package api
 import (
 	"crypto/sha1"
 	"encoding/base64"
-	"log"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
+	log "github.com/go-pkgz/lgr"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 
-	"github.com/umputun/remark/app/rest"
+	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
+	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/app/store"
 )
 
 // GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
@@ -27,16 +27,20 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
+	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Find(locator, sort)
 		if e != nil {
-			return nil, e
+			comments = []store.Comment{} // error should clear comments and continue for post info
 		}
 		maskedComments := s.adminService.alterComments(comments, r)
 		var b []byte
 		switch r.URL.Query().Get("format") {
 		case "tree":
 			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			if tree.Nodes == nil { // eliminate json nil serialization
+				tree.Nodes = []*rest.Node{}
+			}
 			if s.DataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
 			}
@@ -55,7 +59,10 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
 		return
 	}
-	renderJSONFromBytes(w, r, data)
+
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render comments for post %+v", locator)
+	}
 }
 
 // POST /preview, body is a comment, returns rendered html
@@ -78,10 +85,7 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//comment.Text = string(blackfriday.Run([]byte(comment.Text),
+	comment = s.CommentFormatter.Format(comment)
-	//	blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
-	comment.Text = s.ImageProxy.Convert(comment.Text)
 	comment.Sanitize()
 	render.HTML(w, r, comment.Text)
 }
@@ -90,7 +94,8 @@ func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
+	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
 		if e != nil {
 			return nil, e
@@ -103,7 +108,9 @@ func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	renderJSONFromBytes(w, r, data)
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render info for post %+v", locator)
+	}
 }
 
 // GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
@@ -116,22 +123,15 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		limit = 0
 	}
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), func() ([]byte, error) {
+	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Last(siteID, limit)
 		if e != nil {
 			return nil, e
 		}
 		comments = s.adminService.alterComments(comments, r)
-
 		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
-		filterDeleted := []store.Comment{}
+		filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		for _, c := range comments {
-			if c.Deleted {
-				continue
-			}
-			filterDeleted = append(filterDeleted, c)
-		}
-
 		return encodeJSONWithHTML(filterDeleted)
 	})
 
@@ -139,7 +139,10 @@ func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
 		return
 	}
-	renderJSONFromBytes(w, r, data)
+
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render last comments for site %s", siteID)
+	}
 }
 
 // GET /id/{id}?site=siteID&url=post-url - gets a comment by id
@@ -158,7 +161,10 @@ func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
 	render.Status(r, http.StatusOK)
-	renderJSONWithHTML(w, r, comment)
+
+	if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
+		log.Printf("[WARN] can't render last comments for url=%s, id=%s", url, id)
+	}
 }
 
 // GET /comments?site=siteID&user=id - returns comments for given userID
@@ -179,7 +185,8 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), func() ([]byte, error) {
+	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(userID, siteID)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.User(siteID, userID, limit, 0)
 		if e != nil {
 			return nil, e
@@ -198,11 +205,16 @@ func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
 		return
 	}
-	renderJSONFromBytes(w, r, data)
+
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render found comments for user %s", userID)
+	}
 }
 
 // GET /config?site=siteID - returns configuration
 func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
+	siteID := r.URL.Query().Get("site")
+
 	type config struct {
 		Version        string   `json:"version"`
 		EditDuration   int      `json:"edit_duration"`
@@ -219,16 +231,16 @@ func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
 		Version:        s.Version,
 		EditDuration:   int(s.DataService.EditDuration.Seconds()),
 		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.Authenticator.Admins,
+		Admins:         s.DataService.AdminStore.Admins(siteID),
-		AdminEmail:     s.Authenticator.AdminEmail,
+		AdminEmail:     s.DataService.AdminStore.Email(siteID),
 		LowScore:       s.ScoreThresholds.Low,
 		CriticalScore:  s.ScoreThresholds.Critical,
 		ReadOnlyAge:    s.ReadOnlyAge,
 	}
 
 	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers {
+	for _, ap := range s.Authenticator.Providers() {
-		cnf.Auth = append(cnf.Auth, ap.Name)
+		cnf.Auth = append(cnf.Auth, ap.Name())
 	}
 
 	if cnf.Admins == nil { // prevent json serialization to nil
@@ -246,7 +258,7 @@ func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
 		return
 	}
-	render.JSON(w, r, JSON{"count": count, "locator": locator})
+	render.JSON(w, r, R.JSON{"count": count, "locator": locator})
 }
 
 // POST /count?site=siteID - get number of comments for posts from post body
@@ -259,15 +271,15 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// key could be long for multiple posts, make it sha1
-	key := cache.URLKey(r) + strings.Join(posts, ",")
+	k := URLKey(r) + strings.Join(posts, ",")
 	hasher := sha1.New()
-	if _, err := hasher.Write([]byte(key)); err != nil {
+	if _, err := hasher.Write([]byte(k)); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
 		return
 	}
 	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
-
+	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
-	data, err := s.Cache.Get(cache.Key(sha, siteID), func() ([]byte, error) {
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
 		if e != nil {
 			return nil, e
@@ -279,7 +291,10 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
 		return
 	}
-	renderJSONFromBytes(w, r, data)
+
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render comments counters site %s", siteID)
+	}
 }
 
 // GET /list?site=siteID&limit=50&skip=10 - list posts with comments
@@ -295,7 +310,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		skip = v
 	}
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), func() ([]byte, error) {
+	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		posts, e := s.DataService.List(siteID, limit, skip)
 		if e != nil {
 			return nil, e
@@ -307,5 +323,8 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
 		return
 	}
-	renderJSONFromBytes(w, r, data)
+
+	if err = R.RenderJSONFromBytes(w, r, data); err != nil {
+		log.Printf("[WARN] can't render posts lits for site %s", siteID)
+	}
 }

backend/app/rest/api/rest_public_test.go

@@ -9,17 +9,18 @@ import (
 	"testing"
 	"time"
 
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/app/rest"
+	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestRest_Ping(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	res, code := get(t, ts.URL+"/api/v1/ping")
 	assert.Equal(t, "pong", res)
@@ -27,9 +28,8 @@ func TestRest_Ping(t *testing.T) {
 }
 
 func TestRest_Preview(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
 	assert.Nil(t, err)
@@ -40,9 +40,8 @@ func TestRest_Preview(t *testing.T) {
 }
 
 func TestRest_PreviewWithMD(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	require.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	text := `
 # h1
@@ -68,12 +67,15 @@ BKT
 }
 
 func TestRest_Find(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
+	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
+	assert.Equal(t, 200, code)
+	comments := commentsWithInfo{}
+	err := json.Unmarshal([]byte(res), &comments)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(comments.Comments), "should have 0 comments")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -86,10 +88,10 @@ func TestRest_Find(t *testing.T) {
 	assert.NotEqual(t, id1, id2)
 
 	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
+	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
 	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
+	comments = commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
+	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
 	assert.Equal(t, id1, comments.Comments[0].ID)
@@ -122,9 +124,8 @@ func TestRest_Find(t *testing.T) {
 }
 
 func TestRest_FindAge(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
@@ -154,9 +155,8 @@ func TestRest_FindAge(t *testing.T) {
 }
 
 func TestRest_FindReadOnly(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
@@ -174,7 +174,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
+	req.SetBasicAuth("admin", "password")
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
@@ -196,9 +196,12 @@ func TestRest_FindReadOnly(t *testing.T) {
 }
 
 func TestRest_Last(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
+
+	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "[]\n", res, "empty last should return empty list")
 
 	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -210,7 +213,7 @@ func TestRest_Last(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	id2 := addComment(t, c2, ts)
 
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
+	res, code = get(t, ts.URL+"/api/v1/last/2?site=radio-t")
 	assert.Equal(t, 200, code)
 	comments := []store.Comment{}
 	err := json.Unmarshal([]byte(res), &comments)
@@ -233,17 +236,18 @@ func TestRest_Last(t *testing.T) {
 
 	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
 	assert.Nil(t, err)
+	srv.Cache.Flush(cache.FlusherRequest{})
 	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &comments)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, len(comments), "should have 2 comments")
+	t.Logf("%+v", comments)
 }
 
 func TestRest_FindUserComments(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -278,23 +282,20 @@ func TestRest_FindUserComments(t *testing.T) {
 }
 
 func TestRest_UserInfo(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
-	body, code := getWithAuth(t, ts.URL+"/api/v1/user?site=radio-t")
+	body, code := getWithDevAuth(t, ts.URL+"/api/v1/user?site=radio-t")
 	assert.Equal(t, 200, code)
 	user := store.User{}
 	err := json.Unmarshal([]byte(body), &user)
 	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
+	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "http://example.com/pic.png", IP: "127.0.0.1"}, user)
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: ""}, user)
 }
 
 func TestRest_Count(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -309,7 +310,7 @@ func TestRest_Count(t *testing.T) {
 
 	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
 	assert.Equal(t, 200, code)
-	j := JSON{}
+	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
 	assert.Nil(t, err)
 	assert.Equal(t, 3.0, j["count"])
@@ -322,9 +323,8 @@ func TestRest_Count(t *testing.T) {
 }
 
 func TestRest_Counts(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -352,9 +352,8 @@ func TestRest_Counts(t *testing.T) {
 }
 
 func TestRest_List(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	c1 := store.Comment{Text: "test test #1",
 		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
@@ -379,13 +378,12 @@ func TestRest_List(t *testing.T) {
 }
 
 func TestRest_Config(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
 	assert.Equal(t, 200, code)
-	j := JSON{}
+	j := R.JSON{}
 	err := json.Unmarshal([]byte(body), &j)
 	assert.Nil(t, err)
 	assert.Equal(t, 300., j["edit_duration"])
@@ -399,9 +397,8 @@ func TestRest_Config(t *testing.T) {
 }
 
 func TestRest_Info(t *testing.T) {
-	srv, ts := prep(t)
+	ts, srv, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
 
@@ -437,9 +434,8 @@ func TestRest_Info(t *testing.T) {
 }
 
 func TestRest_Robots(t *testing.T) {
-	srv, ts := prep(t)
+	ts, _, teardown := startupT(t)
-	assert.NotNil(t, srv)
+	defer teardown()
-	defer cleanup(ts)
 
 	body, code := get(t, ts.URL+"/robots.txt")
 	assert.Equal(t, 200, code)

backend/app/rest/api/rest_test.go

@@ -0,0 +1,299 @@
+package api
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/auth"
+	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/token"
+	R "github.com/go-pkgz/rest"
+	"github.com/go-pkgz/rest/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest/proxy"
+	"github.com/umputun/remark/backend/app/store"
+	adminstore "github.com/umputun/remark/backend/app/store/admin"
+	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/service"
+)
+
+var testDb = "/tmp/test-remark.db"
+var testHTML = "/tmp/test-remark.html"
+var getStartedHTML = "/tmp/getstarted.html"
+
+var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
+
+func TestRest_FileServer(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	body, code := get(t, ts.URL+"/web/test-remark.html")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "some html", body)
+}
+
+func TestRest_GetStarted(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	err := ioutil.WriteFile(getStartedHTML, []byte("some html blah"), 0700)
+	assert.Nil(t, err)
+
+	body, code := get(t, ts.URL+"/index.html")
+	assert.Equal(t, 200, code)
+	assert.Equal(t, "some html blah", body)
+
+	_ = os.Remove(getStartedHTML)
+	_, code = get(t, ts.URL+"/index.html")
+	assert.Equal(t, 404, code)
+
+}
+
+func TestRest_Shutdown(t *testing.T) {
+	srv := Rest{Authenticator: &auth.Service{}, ImageProxy: &proxy.Image{}}
+
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		srv.Shutdown()
+	}()
+
+	st := time.Now()
+	srv.Run(0)
+	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
+}
+
+func TestRest_filterComments(t *testing.T) {
+	user := store.User{ID: "user1", Name: "user name 1"}
+	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
+	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
+	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
+		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
+
+	r := filterComments([]store.Comment{c1, c2, c3}, func(c store.Comment) bool {
+		return c.Text == "test test #1" || c.Text == "test test #3"
+	})
+	assert.Equal(t, 2, len(r), "one comment filtered")
+}
+
+func TestRest_RunStaticSSLMode(t *testing.T) {
+	srv := Rest{
+		Authenticator: auth.NewService(auth.Opts{
+			AvatarStore:       avatar.NewLocalFS("/tmp"),
+			AvatarResizeLimit: 300,
+		}),
+
+		ImageProxy: &proxy.Image{},
+		SSLConfig: SSLConfig{
+			SSLMode: Static,
+			Port:    8443,
+			Key:     "../../cmd/testdata/key.pem",
+			Cert:    "../../cmd/testdata/cert.pem",
+		},
+		RemarkURL: "https://localhost:8443",
+	}
+
+	go func() {
+		srv.Run(38080)
+	}()
+
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	client := http.Client{
+		// prevent http redirect
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+
+		// allow self-signed certificate
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	resp, err := client.Get("http://localhost:38080/blah?param=1")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 307, resp.StatusCode)
+	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+
+	resp, err = client.Get("https://localhost:8443/ping")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	assert.Equal(t, "pong", string(body))
+
+	srv.Shutdown()
+}
+
+func TestRest_RunAutocertModeHTTPOnly(t *testing.T) {
+	srv := Rest{
+		Authenticator: &auth.Service{},
+		ImageProxy:    &proxy.Image{},
+		SSLConfig: SSLConfig{
+			SSLMode: Auto,
+			Port:    8443,
+		},
+		RemarkURL: "https://localhost:8443",
+	}
+
+	go func() {
+		// can't check https server locally, just only http server
+		srv.Run(38081)
+	}()
+
+	time.Sleep(100 * time.Millisecond) // let server start
+
+	client := http.Client{
+		// prevent http redirect
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	resp, err := client.Get("http://localhost:38081/blah?param=1")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 307, resp.StatusCode)
+	assert.Equal(t, "https://localhost:8443/blah?param=1", resp.Header.Get("Location"))
+
+	srv.Shutdown()
+}
+
+func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
+	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
+	require.Nil(t, err)
+
+	memCache, err := cache.NewMemoryCache()
+	assert.NoError(t, err)
+
+	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
+	restrictedWordsMatcher := service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: []string{"duck"}})
+
+	dataStore := &service.DataStore{
+		Interface:              b,
+		EditDuration:           5 * time.Minute,
+		MaxCommentSize:         4000,
+		AdminStore:             adminStore,
+		MaxVotes:               service.UnlimitedVotes,
+		RestrictedWordsMatcher: restrictedWordsMatcher,
+	}
+
+	srv = &Rest{
+		DataService: dataStore,
+		Authenticator: auth.NewService(auth.Opts{
+			AdminPasswd:  "password",
+			SecretReader: token.SecretFunc(func() (string, error) { return "secret", nil }),
+			AvatarStore:  avatar.NewLocalFS("/tmp/ava-remark42"),
+		}),
+		Cache:     memCache,
+		WebRoot:   "/tmp",
+		RemarkURL: "https://demo.remark42.com",
+
+		ImageProxy:       &proxy.Image{},
+		ReadOnlyAge:      10,
+		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
+		Migrator: &Migrator{
+			DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
+			WordPressImporter: &migrator.WordPress{DataStore: dataStore},
+			NativeImporter:    &migrator.Native{DataStore: dataStore},
+			NativeExporter:    &migrator.Native{DataStore: dataStore},
+			Cache:             &cache.Nop{},
+			KeyStore:          adminStore,
+		},
+	}
+	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
+
+	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
+	assert.Nil(t, err)
+
+	ts = httptest.NewServer(srv.routes())
+
+	teardown = func() {
+		ts.Close()
+		srv.DataService.Close()
+		os.Remove(testDb)
+		os.Remove(testHTML)
+		os.RemoveAll("/tmp/ava-remark42")
+	}
+
+	return ts, srv, teardown
+}
+
+func get(t *testing.T, url string) (string, int) {
+	r, err := http.Get(url)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	require.Nil(t, err)
+	return string(body), r.StatusCode
+}
+
+func getWithDevAuth(t *testing.T, url string) (body string, code int) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("GET", url, nil)
+	require.Nil(t, err)
+	req.Header.Add("X-JWT", devToken)
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	b, err := ioutil.ReadAll(r.Body)
+	assert.Nil(t, err)
+	return string(b), r.StatusCode
+}
+
+func getWithAdminAuth(t *testing.T, url string) (string, int) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("GET", url, nil)
+	require.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	r, err := client.Do(req)
+	require.Nil(t, err)
+	defer r.Body.Close()
+	body, err := ioutil.ReadAll(r.Body)
+	assert.Nil(t, err)
+	return string(body), r.StatusCode
+}
+func post(t *testing.T, url string, body string) (*http.Response, error) {
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", url, strings.NewReader(body))
+	assert.Nil(t, err)
+	req.SetBasicAuth("admin", "password")
+	return client.Do(req)
+}
+
+func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
+	b, err := json.Marshal(c)
+	require.Nil(t, err, "can't marshal comment %+v", c)
+
+	client := &http.Client{Timeout: 5 * time.Second}
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
+	require.Nil(t, err)
+	req.Header.Add("X-JWT", devToken)
+	resp, err := client.Do(req)
+	require.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode)
+	b, err = ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+
+	crResp := R.JSON{}
+	err = json.Unmarshal(b, &crResp)
+	require.Nil(t, err)
+	time.Sleep(time.Nanosecond * 10)
+	return crResp["id"].(string)
+}

backend/app/rest/api/rss.go

@@ -2,19 +2,22 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"time"
 
 	"github.com/go-chi/chi"
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
+	"github.com/pkg/errors"
 
-	"github.com/umputun/remark/app/rest"
+	"github.com/umputun/remark/backend/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
+	"github.com/umputun/remark/backend/app/store"
-	"github.com/umputun/remark/app/store"
 )
 
 const maxRssItems = 20
+const maxLastCommentsReply = 1000
+const maxReplyDuration = 24 * time.Hour
 
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
@@ -23,6 +26,7 @@ func (s *Rest) rssRoutes() chi.Router {
 	router := chi.NewRouter()
 	router.Get("/post", s.rssPostCommentsCtrl)
 	router.Get("/site", s.rssSiteCommentsCtrl)
+	router.Get("/reply", s.rssRepliesCtrl)
 	return router
 }
 
@@ -31,7 +35,8 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
 	log.Printf("[DEBUG] get rss for post %+v", locator)
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
+	key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Find(locator, "-time")
 		if e != nil {
 			return nil, e
@@ -62,7 +67,8 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	siteID := r.URL.Query().Get("site")
 	log.Printf("[DEBUG] get rss for site %s", siteID)
 
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() ([]byte, error) {
+	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
+	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		comments, e := s.DataService.Last(siteID, maxRssItems)
 		if e != nil {
 			return nil, e
@@ -88,6 +94,54 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// GET /rss/reply?user=userID&site=siteID
+func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
+	userID := r.URL.Query().Get("user")
+	siteID := r.URL.Query().Get("site")
+	log.Printf("[DEBUG] get rss replies to user %s for site %s", userID, siteID)
+
+	key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID, lastCommentsScope)
+	data, err := s.Cache.Get(key, func() (res []byte, e error) {
+		comments, e := s.DataService.Last(siteID, maxLastCommentsReply)
+		if e != nil {
+			return nil, errors.Wrap(e, "can't get last comments")
+		}
+		comments = s.adminService.alterComments(comments, r)
+		replies := []store.Comment{}
+		for _, c := range comments {
+			if len(replies) > maxRssItems || c.Timestamp.Add(maxReplyDuration).Before(time.Now()) {
+				break
+			}
+			if c.ParentID != "" && !c.Deleted && c.User.ID != userID { // not interested in replies to yourself
+				var pc store.Comment
+				if pc, e = s.DataService.Get(c.Locator, c.ParentID); e != nil {
+					return nil, errors.Wrap(e, "can't get parent comment")
+				}
+				if pc.User.ID == userID {
+					replies = append(replies, c)
+				}
+			}
+		}
+
+		rss, e := s.toRssFeed(siteID, replies)
+		if e != nil {
+			return nil, e
+		}
+		return []byte(rss), e
+	})
+
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get replies")
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+	w.WriteHeader(http.StatusOK)
+	if _, err := w.Write(data); err != nil {
+		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
+	}
+}
+
 func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 
 	lastCommentTS := time.Unix(0, 0)
@@ -110,6 +164,7 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 			Description: c.Text,
 			Created:     c.Timestamp,
 			Author:      &feeds.Author{Name: c.User.Name},
+			Id:          c.ID,
 		}
 		if c.ParentID != "" {
 			// add indication to parent comment
@@ -120,6 +175,10 @@ func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
 				log.Printf("[WARN] failed to get info about parent comment, %s", err)
 			}
 		}
+		if c.PostTitle != "" {
+			f.Title = f.Title + ", " + c.PostTitle
+		}
+
 		feed.Items = append(feed.Items, &f)
 		if i > maxRssItems {
 			break

backend/app/rest/api/rss_test.go

@@ -0,0 +1,292 @@
+package api
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestServer_RssPost(t *testing.T) {
+	ts, rst, teardown := startupT(t)
+	defer teardown()
+
+	waitOnSecChange()
+
+	c1 := store.Comment{
+		ID:      "1234567890",
+		Text:    "test 123",
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
+	}
+	id1, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	assert.Equal(t, "1234567890", id1)
+	pubDate := time.Now().Format(time.RFC1123Z)
+
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
+	assert.Equal(t, 200, code)
+	t.Log(res)
+
+	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+        <channel>
+            <title>Remark42 comments</title>
+            <link>https://radio-t.com/blah1</link>
+            <description>comment updates</description>
+            <pubDate>%s</pubDate>
+            <item>
+		      <title>developer one</title>
+		      <link>https://radio-t.com/blah1#remark42__comment-1234567890</link>
+		      <description>test 123</description>
+		      <author>developer one</author>
+              <guid>1234567890</guid>
+		      <pubDate>%s</pubDate>
+            </item>
+         </channel>
+	</rss>`, pubDate, pubDate)
+
+	expected, res = cleanRssFormatting(expected, res)
+	assert.Equal(t, expected, res)
+
+	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
+	assert.Equal(t, 400, code)
+}
+
+func TestServer_RssSite(t *testing.T) {
+	ts, rst, teardown := startupT(t)
+	defer teardown()
+
+	waitOnSecChange()
+
+	pubDate := time.Now().Format(time.RFC1123Z)
+
+	c1 := store.Comment{
+		ID:      "comment-id-1",
+		Text:    "test 123",
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
+	}
+	c2 := store.Comment{
+		ID:      "comment-id-2",
+		Text:    "xyz test",
+		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
+	}
+
+	_, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = rst.DataService.Create(c2)
+	require.NoError(t, err)
+
+	require.NoError(t, err)
+	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
+	assert.Equal(t, 200, code)
+	t.Log(res)
+
+	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+		  <channel>
+		    <title>Remark42 comments</title>
+		    <link>radio-t</link>
+		    <description>comment updates</description>
+		    <pubDate>%s</pubDate>
+		    <item>
+		      <title>developer one</title>
+		      <link>https://radio-t.com/blah11#remark42__comment-comment-id-2</link>
+		      <description>xyz test</description>
+		      <author>developer one</author>
+              <guid>comment-id-2</guid>
+		      <pubDate>%s</pubDate>
+		    </item>
+		    <item>
+		      <title>developer one</title>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-1</link>
+		      <description>test 123</description>
+		      <author>developer one</author>
+              <guid>comment-id-1</guid>
+		      <pubDate>%s</pubDate>
+		    </item>
+		  </channel>
+		</rss>`, pubDate, pubDate, pubDate)
+
+	expected, res = cleanRssFormatting(expected, res)
+	assert.Equal(t, expected, res)
+
+	_, code = get(t, ts.URL+"/api/v1/rss/site?site=bad-radio-t")
+	assert.Equal(t, 400, code)
+}
+
+func TestServer_RssWithReply(t *testing.T) {
+	ts, rst, teardown := startupT(t)
+	defer teardown()
+
+	waitOnSecChange()
+
+	pubDate := time.Now().Format(time.RFC1123Z)
+
+	c1 := store.Comment{
+		ID:      "comment-id-1",
+		Text:    "test 123",
+		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:    store.User{ID: "u1", Name: "developer one"},
+	}
+	c2 := store.Comment{
+		ID:       "comment-id-2",
+		ParentID: "comment-id-1",
+		Text:     "xyz test",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
+		User:     store.User{ID: "u1", Name: "developer one"},
+	}
+
+	_, err := rst.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = rst.DataService.Create(c2)
+	require.NoError(t, err)
+
+	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
+	assert.Equal(t, 200, code)
+	t.Log(res)
+
+	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+		  <channel>
+		    <title>Remark42 comments</title>
+		    <link>https://radio-t.com/blah10</link>
+		    <description>comment updates</description>
+		    <pubDate>%s</pubDate>
+		    <item>
+		      <title>developer one &gt; developer one</title>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-2</link>
+		      <description>xyz test</description>
+		      <author>developer one</author>
+              <guid>comment-id-2</guid>
+		      <pubDate>%s</pubDate>
+		    </item>
+		    <item>
+		      <title>developer one</title>
+		      <link>https://radio-t.com/blah10#remark42__comment-comment-id-1</link>
+		      <description>test 123</description>
+		      <author>developer one</author>
+              <guid>comment-id-1</guid>
+		      <pubDate>%s</pubDate>
+		    </item>
+		  </channel>
+		</rss>`, pubDate, pubDate, pubDate)
+
+	expected, res = cleanRssFormatting(expected, res)
+	assert.Equal(t, expected, res)
+}
+
+func TestServer_RssReplies(t *testing.T) {
+	ts, srv, teardown := startupT(t)
+	defer teardown()
+
+	waitOnSecChange()
+
+	pubDate := time.Now().Format(time.RFC1123Z)
+
+	c1 := store.Comment{
+		ID:      "comment-1",
+		Text:    "c1",
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user1"},
+	}
+	c2 := store.Comment{
+		ID:       "comment-2",
+		Text:     "reply to c1 from user2",
+		ParentID: "comment-1",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:     store.User{ID: "user2", Name: "user2"},
+	}
+	c3 := store.Comment{
+		ID:       "comment-3",
+		Text:     "reply to c1 from user3",
+		ParentID: "comment-1",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:     store.User{ID: "user3", Name: "user3"},
+	}
+	c4 := store.Comment{
+		ID:       "comment-4",
+		Text:     "reply to c2 from developer one",
+		ParentID: "comment-2",
+		Locator:  store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:     store.User{ID: "dev", Name: "developer one"},
+	}
+	c5 := store.Comment{
+		ID:      "comment-5",
+		Text:    "developer one",
+		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
+		User:    store.User{ID: "dev", Name: "developer one"},
+	}
+
+	_, err := srv.DataService.Create(c1)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c2)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c3)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c4)
+	require.NoError(t, err)
+	_, err = srv.DataService.Create(c5)
+	require.NoError(t, err)
+
+	// replies to c1 (user1). Must be [c3, c2]
+	res, code := get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t")
+	assert.Equal(t, 200, code)
+	t.Log(res)
+	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+	    <channel>
+	        <title>Remark42 comments</title>
+	        <link>radio-t</link>
+	        <description>comment updates</description>
+	        <pubDate>%s</pubDate>
+	        <item>
+		      <title>user3 &gt; user1</title>
+		      <link>https://radio-t.com/blah1#remark42__comment-comment-3</link>
+		      <description>reply to c1 from user3</description>
+		      <author>user3</author>
+              <guid>comment-3</guid> 
+		      <pubDate>%s</pubDate>
+			</item>
+			<item>
+		      <title>user2 &gt; user1</title>
+		      <link>https://radio-t.com/blah1#remark42__comment-comment-2</link>
+		      <description>reply to c1 from user2</description>
+		      <author>user2</author>
+              <guid>comment-2</guid> 
+		      <pubDate>%s</pubDate>
+			</item>
+	     </channel>
+	</rss>`, pubDate, pubDate, pubDate)
+	expected, res = cleanRssFormatting(expected, res)
+	assert.Equal(t, expected, res)
+
+	_, code = get(t, ts.URL+"/api/v1/rss/reply?user=user1&site=radio-t-bad")
+	assert.Equal(t, 400, code)
+}
+
+func waitOnSecChange() {
+	for {
+		if time.Now().Nanosecond() < 100000000 {
+			break
+		}
+		time.Sleep(10 * time.Nanosecond)
+	}
+}
+
+// clean formatting, i.e. multiple spaces, \t, \n
+func cleanRssFormatting(expected, actual string) (string, string) {
+	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
+
+	expected = strings.Replace(expected, "\n", " ", -1)
+	expected = strings.Replace(expected, "\t", " ", -1)
+	expected = reSpaces.ReplaceAllString(expected, " ")
+
+	actual = strings.Replace(actual, "\n", " ", -1)
+	actual = reSpaces.ReplaceAllString(actual, " ")
+	return expected, actual
+}

backend/app/rest/api/ssl.go

@@ -0,0 +1,131 @@
+package api
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
+	log "github.com/go-pkgz/lgr"
+	"golang.org/x/crypto/acme/autocert"
+
+	R "github.com/go-pkgz/rest"
+)
+
+// sslMode defines ssl mode for rest server
+type sslMode int8
+
+const (
+	// None defines to run http server only
+	None sslMode = iota
+
+	// Static defines to run both https and http server. Redirect http to https
+	Static
+
+	// Auto defines to run both https and http server. Redirect http to https. Https server with autocert support
+	Auto
+)
+
+// SSLConfig holds all ssl params for rest server
+type SSLConfig struct {
+	SSLMode      sslMode
+	Cert         string
+	Key          string
+	Port         int
+	ACMELocation string
+	ACMEEmail    string
+}
+
+// httpToHTTPSRouter creates new router which does redirect from http to https server
+// with default middlewares. Used in 'static' ssl mode.
+func (s *Rest) httpToHTTPSRouter() chi.Router {
+	log.Printf("[DEBUG] create https-to-http redirect routes")
+	router := chi.NewRouter()
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+
+	router.Handle("/*", s.redirectHandler())
+	return router
+}
+
+// httpChallengeRouter creates new router which performs ACME "http-01" challenge response
+// with default middlewares. This part is necessary to obtain certificate from LE.
+// If it receives not a acme challenge it performs redirect to https server.
+// Used in 'auto' ssl mode.
+func (s *Rest) httpChallengeRouter(m *autocert.Manager) chi.Router {
+	log.Printf("[DEBUG] create http-challenge routes")
+	router := chi.NewRouter()
+	router.Use(middleware.RealIP, R.Recoverer(log.Default()))
+	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
+
+	router.Handle("/*", m.HTTPHandler(s.redirectHandler()))
+	return router
+}
+
+func (s *Rest) redirectHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		newURL := s.RemarkURL + r.URL.Path
+		if r.URL.RawQuery != "" {
+			newURL += "?" + r.URL.RawQuery
+		}
+		http.Redirect(w, r, newURL, http.StatusTemporaryRedirect)
+	})
+}
+
+func (s *Rest) makeAutocertManager() *autocert.Manager {
+	return &autocert.Manager{
+		Prompt:     autocert.AcceptTOS,
+		Cache:      autocert.DirCache(s.SSLConfig.ACMELocation),
+		HostPolicy: autocert.HostWhitelist(s.getRemarkHost()),
+		Email:      s.SSLConfig.ACMEEmail,
+	}
+}
+
+// makeHTTPSAutoCertServer makes https server with autocert mode (LE support)
+func (s *Rest) makeHTTPSAutocertServer(port int, router http.Handler, m *autocert.Manager) *http.Server {
+	server := s.makeHTTPServer(port, router)
+	cfg := makeTLSConfig()
+	cfg.GetCertificate = m.GetCertificate
+	server.TLSConfig = cfg
+	return server
+}
+
+// makeHTTPSServer makes https server for static mode
+func (s *Rest) makeHTTPSServer(port int, router http.Handler) *http.Server {
+	server := s.makeHTTPServer(port, router)
+	server.TLSConfig = makeTLSConfig()
+	return server
+}
+
+// getRemarkHost returns hostname for remark server.
+// For example for remarkURL https://remark.com:443 it should return remark.com
+func (s *Rest) getRemarkHost() string {
+	u, err := url.Parse(s.RemarkURL)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}
+
+func makeTLSConfig() *tls.Config {
+	return &tls.Config{
+		PreferServerCipherSuites: true,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			// tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			// tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		},
+		MinVersion: tls.VersionTLS12,
+		CurvePreferences: []tls.CurveID{
+			tls.CurveP256,
+			tls.X25519,
+			tls.CurveP384,
+		},
+	}
+}

backend/app/rest/api/ssl_test.go

@@ -0,0 +1,89 @@
+package api
+
+import (
+	"context"
+	"crypto/tls"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSSL_Redirect(t *testing.T) {
+	rest := Rest{RemarkURL: "https://localhost:443"}
+
+	ts := httptest.NewServer(rest.httpToHTTPSRouter())
+	defer ts.Close()
+
+	client := http.Client{
+		// prevent http redirect
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+
+		// allow self-signed certificate
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	// check http to https redirect response
+	resp, err := client.Get(ts.URL + "/blah?param=1")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 307, resp.StatusCode)
+	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
+}
+
+func TestSSL_ACME_HTTPChallengeRouter(t *testing.T) {
+	rest := Rest{
+		RemarkURL: "https://localhost:443",
+		SSLConfig: SSLConfig{
+			ACMELocation: "acme",
+		},
+	}
+
+	m := rest.makeAutocertManager()
+	defer os.RemoveAll(rest.SSLConfig.ACMELocation)
+
+	ts := httptest.NewServer(rest.httpChallengeRouter(m))
+	defer ts.Close()
+
+	client := http.Client{
+		// prevent http redirect
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	// check http to https redirect response
+	resp, err := client.Get(ts.URL + "/blah?param=1")
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 307, resp.StatusCode)
+	assert.Equal(t, "https://localhost:443/blah?param=1", resp.Header.Get("Location"))
+
+	// check acme http challenge
+	req, err := http.NewRequest("GET", ts.URL+"/.well-known/acme-challenge/token123", nil)
+	require.Nil(t, err)
+	req.Host = "localhost" // for passing hostPolicy check
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 404, resp.StatusCode)
+
+	err = m.Cache.Put(context.Background(), "token123+http-01", []byte("token"))
+	assert.Nil(t, err)
+
+	resp, err = client.Do(req)
+	require.Nil(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, "token", string(body))
+}

backend/app/rest/httperrors.go

@@ -2,13 +2,13 @@ package rest
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"net/url"
 	"runtime"
 	"strings"
 
 	"github.com/go-chi/render"
+	log "github.com/go-pkgz/lgr"
 )
 
 // SendErrorJSON makes {error: blah, details: blah} json body and responds with error code

backend/app/rest/httperrors_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
 )
 
 func TestSendErrorJSON(t *testing.T) {

backend/app/rest/proxy/image.go

@@ -3,15 +3,17 @@ package proxy
 import (
 	"encoding/base64"
 	"io"
-	"log"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/go-chi/chi"
+	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
+
+	"github.com/umputun/remark/backend/app/rest"
 )
 
 // Image extracts image src from comment's html and provides proxy for them
@@ -51,7 +53,7 @@ func (p Image) Routes() chi.Router {
 
 		client := http.Client{Timeout: 30 * time.Second}
 		var resp *http.Response
-		err = retry(5, time.Second, func() error {
+		err = repeater.NewDefault(5, time.Second).Do(func() error {
 			var e error
 			resp, e = client.Get(string(src))
 			return e
@@ -113,7 +115,7 @@ func (p Image) extract(commentHTML string) ([]string, error) {
 	return result, nil
 }
 
-// replace img links in commentHTML with route to proxy with base64 encoded original link
+// replace img links in commentHTML with route to proxy, base64 encoded original link
 func (p Image) replace(commentHTML string, imgs []string) string {
 
 	for _, img := range imgs {

backend/app/rest/testdata/tree.json

@@ -0,0 +1,252 @@
+  {
+      "comments": [{
+              "comment": {
+                  "id": "1",
+                  "pid": "",
+                  "text": "",
+                  "user": {
+                      "name": "",
+                      "id": "",
+                      "picture": "",
+                      "admin": false
+                  },
+                  "locator": {
+                      "site": "site",
+                      "url": "url"
+                  },
+                  "score": 0,
+                  "votes": null,
+                  "time": "2017-12-25T19:46:01Z"
+              },
+              "replies": [{
+                      "comment": {
+                          "id": "11",
+                          "pid": "1",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:46:11Z"
+                      }
+                  },
+                  {
+                      "comment": {
+                          "id": "12",
+                          "pid": "1",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:46:12Z"
+                      }
+                  },
+                  {
+                      "comment": {
+                          "id": "13",
+                          "pid": "1",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:46:13Z"
+                      },
+                      "replies": [{
+                              "comment": {
+                                  "id": "131",
+                                  "pid": "13",
+                                  "text": "",
+                                  "user": {
+                                      "name": "",
+                                      "id": "",
+                                      "picture": "",
+                                      "admin": false
+                                  },
+                                  "locator": {
+                                      "site": "site",
+                                      "url": "url"
+                                  },
+                                  "score": 0,
+                                  "votes": null,
+                                  "time": "2017-12-25T19:46:31Z"
+                              }
+                          },
+                          {
+                              "comment": {
+                                  "id": "132",
+                                  "pid": "13",
+                                  "text": "",
+                                  "user": {
+                                      "name": "",
+                                      "id": "",
+                                      "picture": "",
+                                      "admin": false
+                                  },
+                                  "locator": {
+                                      "site": "site",
+                                      "url": "url"
+                                  },
+                                  "score": 0,
+                                  "votes": null,
+                                  "time": "2017-12-25T19:46:32Z"
+                              }
+                          }
+                      ]
+                  },
+                  {
+                      "comment": {
+                          "id": "14",
+                          "pid": "1",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:46:14Z"
+                      }
+                  }
+              ]
+          },
+          {
+              "comment": {
+                  "id": "2",
+                  "pid": "",
+                  "text": "",
+                  "user": {
+                      "name": "",
+                      "id": "",
+                      "picture": "",
+                      "admin": false
+                  },
+                  "locator": {
+                      "site": "site",
+                      "url": "url"
+                  },
+                  "score": 0,
+                  "votes": null,
+                  "time": "2017-12-25T19:47:02Z"
+              },
+              "replies": [{
+                      "comment": {
+                          "id": "21",
+                          "pid": "2",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:47:21Z"
+                      }
+                  },
+                  {
+                      "comment": {
+                          "id": "22",
+                          "pid": "2",
+                          "text": "",
+                          "user": {
+                              "name": "",
+                              "id": "",
+                              "picture": "",
+                              "admin": false
+                          },
+                          "locator": {
+                              "site": "site",
+                              "url": "url"
+                          },
+                          "score": 0,
+                          "votes": null,
+                          "time": "2017-12-25T19:47:22Z"
+                      }
+                  }
+              ]
+          },
+          {
+              "comment": {
+                  "id": "4",
+                  "pid": "",
+                  "text": "",
+                  "user": {
+                      "name": "",
+                      "id": "",
+                      "picture": "",
+                      "admin": false
+                  },
+                  "locator": {
+                      "site": "site",
+                      "url": "url"
+                  },
+                  "score": 0,
+                  "votes": null,
+                  "time": "2017-12-25T19:47:22Z"
+              }
+          },
+          {
+              "comment": {
+                  "id": "3",
+                  "pid": "",
+                  "text": "",
+                  "user": {
+                      "name": "",
+                      "id": "",
+                      "picture": "",
+                      "admin": false
+                  },
+                  "locator": {
+                      "site": "site",
+                      "url": "url"
+                  },
+                  "score": 0,
+                  "votes": null,
+                  "time": "2017-12-25T19:47:22Z"
+              }
+          }
+      ],
+      "info": {
+          "url": "url",
+          "count": 12,
+          "first_time": "2017-12-25T19:46:01Z",
+          "last_time": "2017-12-25T19:47:22Z"
+      }
+  }

backend/app/rest/testdata/tree_del.json

@@ -0,0 +1,186 @@
+ {
+     "comments": [{
+         "comment": {
+             "id": "1",
+             "pid": "",
+             "text": "",
+             "user": {
+                 "name": "",
+                 "id": "",
+                 "picture": "",
+                 "admin": false
+             },
+             "locator": {
+                 "site": "site",
+                 "url": "url"
+             },
+             "score": 0,
+             "votes": null,
+             "time": "2017-12-25T19:46:01Z"
+         },
+         "replies": [{
+             "comment": {
+                 "id": "11",
+                 "pid": "1",
+                 "text": "",
+                 "user": {
+                     "name": "",
+                     "id": "",
+                     "picture": "",
+                     "admin": false
+                 },
+                 "locator": {
+                     "site": "site",
+                     "url": "url"
+                 },
+                 "score": 0,
+                 "votes": null,
+                 "time": "2017-12-25T19:46:11Z"
+             },
+             "replies": [{
+                 "comment": {
+                     "id": "111",
+                     "pid": "11",
+                     "text": "",
+                     "user": {
+                         "name": "",
+                         "id": "",
+                         "picture": "",
+                         "admin": false
+                     },
+                     "locator": {
+                         "site": "site",
+                         "url": "url"
+                     },
+                     "score": 0,
+                     "votes": null,
+                     "time": "2017-12-25T19:46:12Z"
+                 }
+             }]
+         }]
+     }, {
+         "comment": {
+             "id": "2",
+             "pid": "",
+             "text": "",
+             "user": {
+                 "name": "",
+                 "id": "",
+                 "picture": "",
+                 "admin": false
+             },
+             "locator": {
+                 "site": "site",
+                 "url": "url"
+             },
+             "score": 0,
+             "votes": null,
+             "time": "2017-12-25T19:47:01Z"
+         },
+         "replies": [{
+             "comment": {
+                 "id": "22",
+                 "pid": "2",
+                 "text": "",
+                 "user": {
+                     "name": "",
+                     "id": "",
+                     "picture": "",
+                     "admin": false
+                 },
+                 "locator": {
+                     "site": "site",
+                     "url": "url"
+                 },
+                 "score": 0,
+                 "votes": null,
+                 "time": "2017-12-25T19:47:02Z"
+             },
+             "replies": [{
+                 "comment": {
+                     "id": "223",
+                     "pid": "22",
+                     "text": "",
+                     "user": {
+                         "name": "",
+                         "id": "",
+                         "picture": "",
+                         "admin": false
+                     },
+                     "locator": {
+                         "site": "site",
+                         "url": "url"
+                     },
+                     "score": 0,
+                     "votes": null,
+                     "time": "0001-01-01T00:00:00Z",
+                     "delete": true
+                 },
+                 "replies": [{
+                     "comment": {
+                         "id": "2241",
+                         "pid": "223",
+                         "text": "",
+                         "user": {
+                             "name": "",
+                             "id": "",
+                             "picture": "",
+                             "admin": false
+                         },
+                         "locator": {
+                             "site": "site",
+                             "url": "url"
+                         },
+                         "score": 0,
+                         "votes": null,
+                         "time": "2017-12-25T19:47:05Z"
+                     }
+                 }]
+             }, {
+                 "comment": {
+                     "id": "221",
+                     "pid": "22",
+                     "text": "",
+                     "user": {
+                         "name": "",
+                         "id": "",
+                         "picture": "",
+                         "admin": false
+                     },
+                     "locator": {
+                         "site": "site",
+                         "url": "url"
+                     },
+                     "score": 0,
+                     "votes": null,
+                     "time": "2017-12-25T19:47:03Z"
+                 }
+             }, {
+                 "comment": {
+                     "id": "222",
+                     "pid": "22",
+                     "text": "",
+                     "user": {
+                         "name": "",
+                         "id": "",
+                         "picture": "",
+                         "admin": false
+                     },
+                     "locator": {
+                         "site": "site",
+                         "url": "url"
+                     },
+                     "score": 0,
+                     "votes": null,
+                     "time": "2017-12-25T19:47:04Z"
+                 }
+             }]
+         }]
+     }],
+     "info": {
+         "url": "url",
+         "count": 8,
+         "first_time": "2017-12-25T19:46:01Z",
+         "last_time": "2017-12-25T19:47:05Z"
+     }
+ }

backend/app/rest/tree.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/umputun/remark/app/store"
+	"github.com/umputun/remark/backend/app/store"
 )
 
 // Tree is formatter making tree from the list of comments
@@ -87,18 +87,21 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 
 	repComments := t.filter(comments, func(comment store.Comment) bool { return comment.ParentID == parentID })
 	for _, rc := range repComments {
-		if rc.Timestamp.After(rd.tsModified) {
+		if !rc.Timestamp.IsZero() && rc.Timestamp.After(rd.tsModified) {
 			rd.tsModified = rc.Timestamp
 		}
-		if rc.Timestamp.Before(rd.tsCreated) {
+		if !rc.Timestamp.IsZero() && rc.Timestamp.Before(rd.tsCreated) {
 			rd.tsCreated = rc.Timestamp
 		}
 		if !rc.Deleted {
-			rd.visible = true
+			rd.visible = true // indicates top-level should be visible
 		}
 		rnode := &Node{Comment: rc, Replies: []*Node{}}
 		node.Replies = append(node.Replies, rnode)
 		t.proc(comments, rnode, rd, rc.ID)
+		if !rd.visible || (len(rnode.Replies) == 0 && rc.Deleted) { // clean all-deleted subtree
+			node.Replies = node.Replies[:len(node.Replies)-1]
+		}
 	}
 	// replies always sorted by time
 	sort.Slice(node.Replies, func(i, j int) bool {
@@ -108,8 +111,8 @@ func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentI
 }
 
 // filter returns comments for parentID
-func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) (f []store.Comment) {
+func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) []store.Comment {
-
+	f := []store.Comment{}
 	for _, c := range comments {
 		if fn(c) {
 			f = append(f, c)

backend/app/rest/tree_test.go

@@ -0,0 +1,171 @@
+package rest
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+func TestMakeTree(t *testing.T) {
+
+	loc := store.Locator{URL: "url", SiteID: "site"}
+	ts := func(min int, sec int) time.Time { return time.Date(2017, 12, 25, 19, min, sec, 0, time.UTC) }
+
+	// unsorted by purpose
+	comments := []store.Comment{
+		{Locator: loc, ID: "14", ParentID: "1", Timestamp: ts(46, 14)},
+		{Locator: loc, ID: "1", Timestamp: ts(46, 1)},
+		{Locator: loc, ID: "2", Timestamp: ts(47, 2)},
+		{Locator: loc, ID: "11", ParentID: "1", Timestamp: ts(46, 11)},
+		{Locator: loc, ID: "13", ParentID: "1", Timestamp: ts(46, 13)},
+		{Locator: loc, ID: "12", ParentID: "1", Timestamp: ts(46, 12)},
+		{Locator: loc, ID: "131", ParentID: "13", Timestamp: ts(46, 31)},
+		{Locator: loc, ID: "132", ParentID: "13", Timestamp: ts(46, 32)},
+		{Locator: loc, ID: "21", ParentID: "2", Timestamp: ts(47, 21)},
+		{Locator: loc, ID: "22", ParentID: "2", Timestamp: ts(47, 22)},
+		{Locator: loc, ID: "4", Timestamp: ts(47, 22)},
+		{Locator: loc, ID: "3", Timestamp: ts(47, 22)},
+		{Locator: loc, ID: "5", Deleted: true},
+		{Locator: loc, ID: "6", Deleted: true},
+		{Locator: loc, ID: "61", ParentID: "6", Deleted: true},
+		{Locator: loc, ID: "62", ParentID: "6", Deleted: true},
+		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
+	}
+
+	res := MakeTree(comments, "time", 0)
+	resJSON, err := json.Marshal(&res)
+	require.Nil(t, err)
+
+	expJSON := mustLoadJSONFile(t, "testdata/tree.json")
+	assert.Equal(t, expJSON, resJSON)
+	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
+
+	res = MakeTree([]store.Comment{}, "time", 0)
+	assert.Equal(t, &Tree{}, res)
+
+	res = MakeTree(comments, "time", 10)
+	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
+}
+
+func TestMakeEmptySubtree(t *testing.T) {
+	loc := store.Locator{URL: "url", SiteID: "site"}
+	ts := func(min int, sec int) time.Time { return time.Date(2017, 12, 25, 19, min, sec, 0, time.UTC) }
+
+	// unsorted by purpose
+	comments := []store.Comment{
+		{Locator: loc, ID: "1", Timestamp: ts(46, 1)},
+		{Locator: loc, ID: "11", ParentID: "1", Timestamp: ts(46, 11)},
+		{Locator: loc, ID: "111", ParentID: "11", Timestamp: ts(46, 12)},
+		{Locator: loc, ID: "112", ParentID: "11", Deleted: true}, // subtree deleted
+		{Locator: loc, ID: "1121", ParentID: "112", Deleted: true},
+		{Locator: loc, ID: "1122", ParentID: "112", Deleted: true},
+		{Locator: loc, ID: "12", ParentID: "12", Deleted: true}, // subcomment deleted
+
+		{Locator: loc, ID: "2", Timestamp: ts(47, 1)},
+		{Locator: loc, ID: "21", ParentID: "2", Deleted: true}, // subtree deleted
+		{Locator: loc, ID: "211", ParentID: "21", Deleted: true},
+		{Locator: loc, ID: "212", ParentID: "21", Deleted: true},
+		{Locator: loc, ID: "22", ParentID: "2", Timestamp: ts(47, 2)},
+		{Locator: loc, ID: "221", ParentID: "22", Timestamp: ts(47, 3)},
+		{Locator: loc, ID: "222", ParentID: "22", Timestamp: ts(47, 4)},
+		{Locator: loc, ID: "223", ParentID: "22", Deleted: true},
+		{Locator: loc, ID: "224", ParentID: "22", Deleted: true},
+		{Locator: loc, ID: "2241", ParentID: "223", Timestamp: ts(47, 5)},
+		{Locator: loc, ID: "3", Timestamp: ts(48, 1), Deleted: true}, // deleted top level
+	}
+
+	res := MakeTree(comments, "time", 0)
+	resJSON, err := json.Marshal(&res)
+	require.Nil(t, err)
+	log.Print(string(resJSON))
+
+	expJSON := mustLoadJSONFile(t, "testdata/tree_del.json")
+	assert.Equal(t, string(expJSON), string(resJSON))
+
+}
+
+func TestTreeSortNodes(t *testing.T) {
+	// unsorted by purpose
+	comments := []store.Comment{
+		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
+		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
+		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
+		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
+		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
+		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
+		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
+		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
+		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
+		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
+		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
+		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
+		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
+		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
+	}
+
+	res := MakeTree(comments, "+active", 0)
+	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
+	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
+
+	res = MakeTree(comments, "-active", 0)
+	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+
+	res = MakeTree(comments, "+time", 0)
+	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+
+	res = MakeTree(comments, "-time", 0)
+	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
+
+	res = MakeTree(comments, "score", 0)
+	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
+
+	res = MakeTree(comments, "+score", 0)
+	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
+
+	res = MakeTree(comments, "-score", 0)
+	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
+	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
+	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
+	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
+
+	res = MakeTree(comments, "undefined", 0)
+	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
+	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
+}
+
+func BenchmarkTree(b *testing.B) {
+	comments := []store.Comment{}
+	data, err := ioutil.ReadFile("testdata/tree_bench.json")
+	assert.Nil(b, err)
+	err = json.Unmarshal(data, &comments)
+	assert.Nil(b, err)
+
+	for i := 0; i < b.N; i++ {
+		res := MakeTree(comments, "time", 0)
+		assert.NotNil(b, res)
+	}
+}
+
+// loadJsonFile read fixtrue file and clear any custom json formatting
+func mustLoadJSONFile(t *testing.T, file string) []byte {
+	expJSON, err := ioutil.ReadFile(file)
+	require.Nil(t, err)
+	expTree := Tree{}
+	err = json.Unmarshal(expJSON, &expTree)
+	require.Nil(t, err)
+	expJSON, err = json.Marshal(expTree)
+	require.Nil(t, err)
+	return expJSON
+}

backend/app/rest/user.go

@@ -0,0 +1,57 @@
+package rest
+
+import (
+	"net/http"
+
+	"github.com/go-pkgz/auth/token"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark/backend/app/store"
+)
+
+// MustGetUserInfo fails if can't extract user data from the request.
+// should be called from authed controllers only
+func MustGetUserInfo(r *http.Request) store.User {
+	user, err := GetUserInfo(r)
+	if err != nil {
+		panic(err)
+	}
+	return user
+}
+
+// GetUserInfo returns user from request context
+func GetUserInfo(r *http.Request) (user store.User, err error) {
+
+	u, err := token.GetUserInfo(r)
+	if err != nil {
+		return store.User{}, errors.Wrap(err, "can't extract user info from the token")
+	}
+
+	return store.User{
+		Name:     u.Name,
+		ID:       u.ID,
+		IP:       u.IP,
+		Picture:  u.Picture,
+		Admin:    u.IsAdmin(),
+		Verified: u.BoolAttr("verified"),
+		Blocked:  u.BoolAttr("blocked"),
+	}, nil
+
+}
+
+// SetUserInfo sets user into request context
+func SetUserInfo(r *http.Request, user store.User) *http.Request {
+	u := token.User{
+		ID:      user.ID,
+		Name:    user.Name,
+		Picture: user.Picture,
+		IP:      user.IP,
+		Attributes: map[string]interface{}{
+			"blocked":  user.Blocked,
+			"verified": user.Verified,
+		},
+	}
+	u.SetAdmin(user.Admin)
+
+	return token.SetUserInfo(r, u)
+}