move back css for last comments

with zh.json and en.json.

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
-/web/public/
+/frontend/node_modules/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/objects/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,3 +19,6 @@ debug
 debug.test
 *.prof
 *.test
+remark42
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -1,80 +1,105 @@
-workspace:
-  base: /go
-  path: src/github.com/umputun/remark
+kind: pipeline
+name: default
+type: docker
 
-pipeline:
-  
-  build:
-    image: golang:1.10-alpine
+steps:
+
+  - name: build server
+    image: umputun/baseimage:buildgo-latest
     commands:
-      - cd app
-      - go build -v ./...
+      - cd backend/app
+      - go build -mod=vendor
+      - echo "build completed"
 
-  docker_master:
+  - name: docker master
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    tags: 
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
     when:
-      branch: [master, release/*]
+      branch: [master]
       event: push
 
-  docker_tag:
+  - name: docker tag
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-    tags: 
-      - ${DRONE_TAG}
-      - latest
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+      tags:
+        - ${DRONE_TAG}
+        - latest
     when:
       event: tag
 
-  docker_branch:
+  - name: docker branch
     image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    tags: 
-      - ${DRONE_COMMIT_BRANCH/\//-}
+    settings:
+      repo: umputun/remark42
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - DRONE_BRANCH=${DRONE_BRANCH}
+      tags:
+        - ${DRONE_COMMIT_BRANCH/\//-}
+      dry_run: true
     when:
       branch:
         exclude: [master, release/*]
         event: push
 
-  docker_pullrequest:
-    image: docker
-    commands:
-      - docker build .
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+  - name: artifacts tag
+    image: plugins/docker
+    settings:
+      dockerfile: Dockerfile.artifacts
+      build_args:
+        - DRONE=${DRONE}
+        - DRONE_TAG=${DRONE_TAG}
+        - DRONE_COMMIT=${DRONE_COMMIT}
+        - GITHUB_TOKEN=${GITHUB_TOKEN}
     when:
-      event: pull_request
+      event: tag
 
-  deploy:
-    image: appleboy/drone-ssh
-    host: remark42.com
-    username: umputun
-    port: 22
-    secrets: [ ssh_key ]
-    script:
-      - cd /srv/remark
-      - docker-compose pull
-      - docker-compose up -d
+  - name: deploy
+    image: docker.umputun.com/system/deploy-ci:master
+    commands:
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose pull"
+      - ssh umputun@remark42.com "cd /srv/remark && docker-compose up -d"
+      - ssh umputun@remark42.com "cd /srv/remark-site && git pull && git submodule update --recursive --remote"
+      - ssh umputun@remark42.com "cd /srv/remark-site && docker-compose build && docker-compose up"
     when:
       branch: master
       event: push
 
-  notify:
+  - name: notify
     image: drillster/drone-email
-    host: smtp.mailgun.org
-    port: 25
-    username: email_username
-    password: email_password
-    from: drone@mg.umputun.com
-    recipients: [ umputun@gmail.com ]
-    secrets: [ email_username, email_password ]
+    settings:
+      host: smtp.mailgun.org
+      username:
+        from_secret: email_username
+      password:
+        from_secret: email_password
+      from: drone@mg.umputun.dev
+      recipients: [ sys@umputun.dev ]
     when:
       status: [ changed, failure ]

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*       @umputun

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+patreon: remark42

.github/workflows/ci-build.yml

@@ -0,0 +1,31 @@
+name: build
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-build.yml'
+      - 'backend/**'
+      - 'frontend/**'
+      - '.dockerignore'
+      - 'docker-init.sh'
+      - 'Dockerfile'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: build docker image
+        run: docker build --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github .

.github/workflows/ci-frontend-size-limit.yml

@@ -0,0 +1,19 @@
+name: frontend
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/ci-frontend-size-limit.yml'
+      - 'frontend/**'
+
+jobs:
+  size:
+    runs-on: ubuntu-latest
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - uses: actions/checkout@v1
+      - uses: andresz1/size-limit-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          directory: frontend

.github/workflows/ci-frontend.yml

@@ -0,0 +1,105 @@
+name: frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-frontend.yml'
+      - 'frontend/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-frontend.yml'
+      - 'frontend/**'
+
+jobs:
+  check-transtations:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:translation
+        working-directory: ./frontend
+
+  check-typescript:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:types
+        working-directory: ./frontend
+
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npx run-p lint
+        working-directory: ./frontend
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npm run test:coverage
+        working-directory: ./frontend
+
+      - name: submit coverage
+        run: node ${{ github.workspace }}/frontend/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-test-backend.yml

@@ -0,0 +1,63 @@
+name: test_backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+  pull_request:
+    paths:
+      - '.github/workflows/ci-test-backend.yml'
+      - 'backend/**'
+      - '!backend/scripts/**'
+
+jobs:
+  backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: debug if needed
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.14
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.26.0
+          go get -u github.com/mattn/goveralls
+
+      - name: test and lint backend
+        run: |
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          GOFLAGS: "-mod=vendor"
+          TZ: "America/Chicago"
+
+      - name: test and lint examples
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
+
+      - name: submit coverage
+        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -8,9 +8,19 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
-/web/public/
+/frontend/node_modules/
+/frontend/public/
+/frontend/coverage
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
+.mongo
+remark42
+/bin/
+/backend/var/
+/backend/app/var/
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor

.travis.yml

@@ -1,19 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      .

Dockerfile

@@ -1,77 +1,82 @@
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM umputun/baseimage:buildgo-v1.6.1 as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
+ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+ARG DRONE_PULL_REQUEST
 
-WORKDIR /go/src/github.com/umputun/remark
+ARG SKIP_BACKEND_TEST
+ARG BACKEND_TEST_TIMEOUT
 
-ADD app /go/src/github.com/umputun/remark/app
-ADD vendor /go/src/github.com/umputun/remark/vendor
-ADD .git /go/src/github.com/umputun/remark/.git
+ADD backend /build/backend
+ADD .git/ /build/backend/.git/
+WORKDIR /build/backend
 
-RUN cd app && go test ./...
+ENV GOFLAGS="-mod=vendor"
 
-RUN gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./...
-
-RUN mkdir -p target && /script/coverage.sh
-
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo coverall not enabled ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi
+# install gcc in order to be able to go test package with -race
+RUN apk --no-cache add gcc libc-dev
 
+# run tests
 RUN \
-    version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) && \
-    echo "git version=$version" && \  
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "runs outside of drone" ; \
-    else version=${DRONE_TAG}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
-    echo "final version=$version" && \  
-    go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app
+    cd app && \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        golangci-lint run --config ../.golangci.yml ./... ; \
+    else echo "skip backend tests and linter" ; fi
 
+# if DRONE presented use DRONE_* git env to make version
+RUN \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
+    echo "version=$version" && \
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM node:9.4-alpine as build-frontend
+FROM node:12.16-alpine as build-frontend-deps
+
+ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
-ADD web /srv/web
 RUN apk add --no-cache --update git
-RUN \
-    cd /srv/web && \
-    npm i && npm run build && \
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci --loglevel warn
+
+FROM node:12.16-alpine as build-frontend
+
+ARG CI
+ARG SKIP_FRONTEND_TEST
+ARG NODE_ENV=production
+
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
+    else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-
-FROM umputun/baseimage:app-latest
+FROM umputun/baseimage:app-v1.6.1
 
 WORKDIR /srv
 
-ADD scripts/import-disqus.sh /srv/import-disqus.sh
-ADD scripts/restore-backup.sh /srv/restore-backup.sh
-ADD scripts/migrate-data.sh /srv/migrate-data.sh
-ADD scripts/create-backup.sh /srv/create-backup.sh
+ADD docker-init.sh /entrypoint.sh
+ADD backend/scripts/backup.sh /usr/local/bin/backup
+ADD backend/scripts/restore.sh /usr/local/bin/restore
+ADD backend/scripts/import.sh /usr/local/bin/import
+RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-ADD start.sh /srv/start.sh
-
-RUN chmod +x /srv/start.sh /srv/import-disqus.sh /srv/restore-backup.sh /srv/migrate-data.sh /srv/create-backup.sh
-
-COPY --from=build-backend /go/src/github.com/umputun/remark/remark /srv/
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-backend /build/backend/templates /srv
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
 RUN chown -R app:app /srv
+RUN ln -s /srv/remark42 /usr/bin/remark42
 
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["/srv/start.sh"]
-ENTRYPOINT ["/init.sh"]
+COPY docker-init.sh /srv/init.sh
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -0,0 +1,105 @@
+FROM node:12.16-alpine as build-frontend-deps
+
+ARG CI
+ARG DRONE
+ARG DRONE_TAG
+ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+
+ENV SKIP_FRONTEND_TEST=true
+
+RUN apk add --no-cache --update git
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
+
+FROM node:12.16-alpine as build-frontend
+
+ARG CI
+ARG NODE_ENV=production
+ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
+
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
+    npm run build && \
+    rm -rf ./node_modules
+
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+ARG GITHUB_TOKEN
+ENV SKIP_BACKEND_TEST=true
+
+WORKDIR /build/backend
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+ADD .git/ /build/backend/.git/
+
+COPY --from=build-frontend /srv/frontend/public/ web
+
+RUN \
+    export WEB_ROOT=/build/backend/web && \
+    find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \
+    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
+    statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \
+    ls -la /build/backend/app/templates/statik.go && \
+    ls -la /build/backend/app/rest/api/statik.go && \
+    ls -la /build/backend/web/
+
+# if DRONE presented use DRONE_* git env to make version
+RUN \
+    if [ -z "$DRONE" ] ; then \
+    echo "runs outside of drone" && version=$(/script/git-rev.sh); \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    echo "version=$version" && \
+    export GOFLAGS="-mod=vendor" && \
+    GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+
+RUN \
+    if [ -z "$DRONE_TAG" ] ; then \
+    echo "runs outside of drone" && tag=""; \
+    else tag=_${DRONE_TAG}; fi && \
+    apk add --no-cache --update zip && \
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
+
+# upload to github
+#RUN \
+#    if [ -z "$DRONE_TAG" ] ; then \
+#    echo "skip upload to github" ; \
+#    else \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-386.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-386.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.linux-arm64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.linux-arm64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/gzip" --data-binary @remark42_${DRONE_TAG}.darwin-amd64.tar.gz \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.darwin-amd64.tar.gz" && \
+#    curl -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.manifold-preview" \
+#    -H "Content-Type: application/zip" --data-binary @remark42_${DRONE_TAG}.windows-amd64.zip \
+#    "https://uploads.github.com/repos/umputun/remark/releases/${DRONE_TAG}/assets?name=remark_${DRONE_TAG}.windows-amd64.zip"; fi
+
+FROM alpine
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
+RUN ls -la /artifacts/*
+CMD ["sleep", "100"]

Gopkg.lock

@@ -1,237 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter"
-  ]
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware"
-  ]
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/google/uuid"
-  packages = ["."]
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru"
-  ]
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-  packages = ["."]
-  revision = "0dc08b1671f34c4250ce212759ebd880f743d883"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64"
-  ]
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom"
-  ]
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex"
-  ]
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch"
-  ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "af8b7f1817ce6e82746722a745184bd50c341733bcc024311aa956ea32475796"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,36 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2020 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,41 @@
+OS=linux
+ARCH=amd64
+
+bin:
+	docker build -f Dockerfile.artifacts -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
+	docker rm -f remark42.bin
+
+docker:
+	docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
+
+deploy:
+	docker build -f Dockerfile.artifacts -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	- @mkdir -p bin
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
+	docker rm -f remark42.bin
+
+race_test:
+	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	docker pull umputun/baseimage:buildgo-latest
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,78 +1,235 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://codecov.io/gh/umputun/remark42)
+
+
+
 
 Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, Microsoft, GitHub and Yandex
+* Login via email
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus
-* Markdown support
+* Import from Disqus and WordPress
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
+* Telegram and email notifications
 * Export data to json with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
-* Clean, lightweight and fully customizable UI
+* Self-contained executable can be deployed directly to Linux, Windows and MacOS
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
-* Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le)
+* Integration with automatic ssl (direct and via [nginx-le](https://github.com/umputun/nginx-le))
 * [Privacy focused](#privacy)
 
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
+
+<details><summary>Screenshots</summary>
+
+Comments example:
+![](https://github.com/umputun/remark42/blob/master/screenshots/comments.png)
+
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark42/wiki/Admin-UI)
+</details>
+
+
+#
+
+  - [Install](#install)
+    - [Backend](#backend)
+      - [With Docker](#with-docker)
+      - [Without Docker](#without-docker)
+      - [Parameters](#parameters)
+        - [Required parameters](#required-parameters)
+      - [Quick installation test](#quick-installation-test)
+      - [Register oauth2 providers](#register-oauth2-providers)
+        - [Google Auth Provider](#google-auth-provider)
+        - [GitHub Auth Provider](#github-auth-provider)
+        - [Facebook Auth Provider](#facebook-auth-provider)
+        - [Twitter Auth Provider](#twitter-auth-provider)
+        - [Yandex Auth Provider](#yandex-auth-provider)
+      - [Initial import from Disqus](#initial-import-from-disqus)
+      - [Initial import from WordPress](#initial-import-from-wordpress)
+      - [Backup and restore](#backup-and-restore)
+        - [Automatic backups](#automatic-backups)
+        - [Manual backup](#manual-backup)
+        - [Restore from backup](#restore-from-backup)
+        - [Backup format](#backup-format)
+      - [Admin users](#admin-users)
+    - [Setup on your website](#setup-on-your-website)
+      - [Comments](#comments)
+      - [Last comments](#last-comments)
+      - [Counter](#counter)
+  - [Build from the source](#build-from-the-source)
+  - [Development](#development)
+    - [Backend development](#backend-development)
+    - [Frontend development](#frontend-development)
+      - [Build](#build)
+      - [Devserver](#devserver)
+  - [API](#api)
+    - [Authorization](#authorization)
+    - [Commenting](#commenting)
+    - [RSS feeds](#rss-feeds)
+    - [Admin](#admin)
+  - [Privacy](#privacy)
+  - [Technical details](#technical-details)
+
+
 ## Install
 
 ### Backend
 
+#### With Docker
+
+_this is the recommended way to run remark42_
+
 * copy provided `docker-compose.yml` and customize for your needs
-* prepare user id for container `` export USER=`id -u $USER` ``
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
-* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
-* alternatively compile from sources - `docker-compose build && docker-compose up -d`
+* make sure you **don't keep** `ADMIN_PASSWD=something...` for any non-development deployments
+* pull prepared images from the DockerHub and start - `docker-compose pull && docker-compose up -d`
+* alternatively compile from the sources - `docker-compose build && docker-compose up -d`
+
+#### Without Docker
+
+* download archive for [stable release](https://github.com/umputun/remark42/releases) or [development version](https://remark42.com/downloads)
+* unpack with `gunzip` (Linux, macOS) or with `zip` (Windows)
+* run as `remark42.{os}-{arch} server {parameters...}`, i.e. `remark42.linux-amd64 server --secret=12345 --url=http://127.0.0.1:8080`
+* alternatively compile from the sources - `make OS=[linux|darwin|windows] ARCH=[amd64,386,arm64,arm]`
 
 #### Parameters
 
-| Command line       | Environment        | Default               | Description                                    |
-| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
-| url                | REMARK_URL         |                       | url to remark42 server, _required_             |
-| secret             | SECRET             |                       | secret key, _required_                         |
-| store.bolt.path    | STORE_BOLT_PATH    | `./var`               | path to data directory                         |
-| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s`                 | boltdb access timeout                          |
-| site               | SITE               | `remark`              | site name(s), _multi_                          |
-| admin              | ADMIN              |                       | admin names (list of user ids), _multi_        |
-| admin-email        | ADMIN_EMAIL        | `admin@${REMARK_URL}` | admin email                                    |
-| backup             | BACKUP_PATH        | `./var/backup`        | backups location                               |
-| max-back           | MAX_BACKUP_FILES   | `10`                  | max backup files to keep                       |
-| cache.max.items    | CACHE_MAX_ITEMS    | `1000`                | max number of cached items, `0` - unlimited    |
-| cache.max.value    | CACHE_MAX_VALUE    | `65536`               | max size of cached value, `0` - unlimited      |
-| cache.max.size     | CACHE_MAX_SIZE     | `50000000`            | max size of all cached values, `0` - unlimited |
-| avatar.path        | AVATAR_FS_PATH     | `./var/avatars`       | avatars location                               |
-| avatar.rsz-lmt     | AVATAR_RSZ_LMT     | 0                     | max image size for resizing avatars on save    |
-| max-comment        | MAX_COMMENT_SIZE   | 2048                  | comment's size limit                           |
-| auth.google.cid    | AUTH_GOOGLE_CID    |                       | Google OAuth client ID                         |
-| auth.google.csec   | AUTH_GOOGLE_CSEC   |                       | Google OAuth client secret                     |
-| auth.facebook.cid  | AUTH_FACEBOOK_CID  |                       | Facebook OAuth client ID                       |
-| auth.facebook.csec | AUTH_FACEBOOK_CSEC |                       | Facebook OAuth client secret                   |
-| auth.github.cid    | AUTH_GITHUB_CID    |                       | Github OAuth client ID                         |
-| auth.github.csec   | AUTH_GITHUB_CSEC   |                       | Github OAuth client secret                     |
-| auth.yandex.cid    | AUTH_YANDEX_CID    |                       | Yandex OAuth client ID                         |
-| auth.yandex.csec   | AUTH_YANDEX_CSEC   |                       | Yandex OAuth client secret                     |
-| low-score          | LOW_SCORE          | `-5`                  | Low score threshold                            |
-| critical-score     | CRITICAL_SCORE     | `-10`                 | Critical score threshold                       |
-| img-proxy          | IMG_PROXY          | `false`               | Enable http->https proxy for images            |
-| dbg                | DEBUG              | `false`               | debug mode                                     |
-| dev-passwd         | DEV_PASSWD         |                       | password for `dev` user                        |
+| Command line            | Environment             | Default                  | Description                                     |
+| ----------------------- | ----------------------- | ------------------------ | ----------------------------------------------- |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_              |
+| secret                  | SECRET                  |                          | secret key, _required_                          |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                           |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `rpc`                |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                          |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                           |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin ids (list of user ids), _multi_           |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin emails, _multi_                           |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                        |
+| cache.type              | CACHE_TYPE              | `mem`                    | type of cache, `redis_pub_sub` or `mem` or `none` |
+| cache.redis_addr        | CACHE_REDIS_ADDR        | `127.0.0.1:6379`         | address of redis PubSub instance, turn `redis_pub_sub` cache on for distributed cache |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited     |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited       |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited  |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `uri`  |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                 |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                     |
+| avatar.uri              | AVATAR_URI              | `./var/avatars`          | avatar store uri                                |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save     |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, `bolt`             |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                          |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                    |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                      |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                      |
+| image.bolt.file         | IMAGE_BOLT_FILE         | `/var/pictures.db`       | images bolt file location                       |
+| image.resize-width      | IMAGE_RESIZE_WIDTH      | `2400`                   | width of resized image                          |
+| image.resize-height     | IMAGE_RESIZE_HEIGHT     | `900`                    | height of resized image                         |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                         |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                      |
+| auth.send-jwt-header    | AUTH_SEND_JWT_HEADER    | `false`                  | send JWT as a header instead of cookie          |
+| auth.same-site          | AUTH_SAME_SITE          | `default`                | set same site policy for cookies (`default`, `none`, `lax` or `strict`)|
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                          |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                      |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                        |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                    |
+| auth.microsoft.cid      | AUTH_MICROSOFT_CID      |                          | Microsoft OAuth client ID                       |
+| auth.microsoft.csec     | AUTH_MICROSOFT_CSEC     |                          | Microsoft OAuth client secret                   |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | GitHub OAuth client ID                          |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | GitHub OAuth client secret                      |
+| auth.twitter.cid        | AUTH_TWITTER_CID        |                          | Twitter Consumer API Key                        |
+| auth.twitter.csec       | AUTH_TWITTER_CSEC       |                          | Twitter Consumer API Secret key                 |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                          |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                      |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only      |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                          |
+| auth.email.enable       | AUTH_EMAIL_ENABLE       | `false`                  | enable auth via email                           |
+| auth.email.from         | AUTH_EMAIL_FROM         |                          | email from                                      |
+| auth.email.subj         | AUTH_EMAIL_SUBJ         | `remark42 confirmation`  | email subject                                   |
+| auth.email.content-type | AUTH_EMAIL_CONTENT_TYPE | `text/html`              | email content type                              |
+| auth.email.template     | AUTH_EMAIL_TEMPLATE     | none (predefined)        | custom email message template file              |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (telegram and/or email)    |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                      |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                  |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                |
+| notify.email.fromAddress | NOTIFY_EMAIL_FROM      |                          | from email address                              |
+| notify.email.verification_subj | NOTIFY_EMAIL_VERIFICATION_SUBJ | `Email verification` | verification message subject          |
+| notify.email.notify_admin | NOTIFY_EMAIL_ADMIN    | `false`                  | notify admin on new comments via ADMIN_SHARED_EMAIL |
+| smtp.host               | SMTP_HOST               |                          | SMTP host                                       |
+| smtp.port               | SMTP_PORT               |                          | SMTP port                                       |
+| smtp.username           | SMTP_USERNAME           |                          | SMTP user name                                  |
+| smtp.password           | SMTP_PASSWORD           |                          | SMTP password                                   |
+| smtp.tls                | SMTP_TLS                |                          | enable TLS for SMTP                             |
+| smtp.timeout            | SMTP_TIMEOUT            | `10s`                    | SMTP TCP connection timeout                     |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le  |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                           |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                           |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                            |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored      |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                            |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited       |
+| votes-ip                | VOTES_IP                | `false`                  | restrict votes from the same ip                 |
+| anon-vote               | ANON_VOTE               | `false`                  | allow voting for anonymous users, require VOTES_IP to be enabled as well |
+| votes-ip-time           | VOTES_IP_TIME           | `5m`                     | same ip vote restriction time, `0s` - unlimited |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                             |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                        |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive   |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_ |
+| restricted-names        | RESTRICTED_NAMES        |                          | names prohibited to use by the user, _multi_    |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                     |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                 |
+| image-proxy.http2https  |  IMAGE_PROXY_HTTP2HTTPS | `false`                  | enable http->https proxy for images             |
+| image-proxy.cache-external | IMAGE_PROXY_CACHE_EXTERNAL | `false`            | enable caching external images to current image storage |
+| emoji                   | EMOJI                   | `false`                  | enable emoji support                            |
+| simple-view             | SIMPLE_VIEW             | `false`                  | minimized UI with basic info only               |
+| proxy-cors              | PROXY_CORS              | `false`                  | disable internal CORS and delegate it to proxy  |
+| allowed-hosts           | ALLOWED_HOSTS           |  enable all              | limit hosts/sources allowed to embed comments   |
+| port                    | REMARK_PORT             | `8080`                   | web server port                                 |
+| web-root                | REMARK_WEB_ROOT         | `./web`                  | web server root directory                       |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                               |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                 |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                      |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
 * _required_ parameters have to be presented in the environment or provided in command line
 
+##### Deprecated
+
+Following list of command-line options is deprecated and will be removed in 2 minor releases or 1 major release (whichever is closer)
+from the version in which they were deprecated. After remark42 version update, please check startup log once for deprecation warnings to avoid
+trouble with unrecognized command-line options in the future.
+
+<details>
+<summary>deprecated options</summary>
+
+| Command line       | Replacement   | Environment        | Replacement   | Default | Description    | Deprecation version |
+| ------------------ | ------------- | ------------------ | ------------- | ------- | -------------- | ------------------- |
+| auth.email.host    | smtp.host     | AUTH_EMAIL_HOST    | SMTP_HOST     |         | smtp host      | 1.5.0               |
+| auth.email.port    | smtp.port     | AUTH_EMAIL_PORT    | SMTP_PORT     |         | smtp port      | 1.5.0               |
+| auth.email.user    | smtp.username | AUTH_EMAIL_USER    | SMTP_USERNAME |         | smtp user name | 1.5.0               |
+| auth.email.passwd  | smtp.password | AUTH_EMAIL_PASSWD  | SMTP_PASSWORD |         | smtp password  | 1.5.0               |
+| auth.email.tls     | smtp.tls      | AUTH_EMAIL_TLS     | SMTP_TLS      | `false` | enable TLS     | 1.5.0               |
+| auth.email.timeout | smtp.timeout  | AUTH_EMAIL_TIMEOUT | SMTP_TIMEOUT  | `10s`   | smtp timeout   | 1.5.0               |
+| img-proxy          | image-proxy.http2https | IMG_PROXY | IMAGE_PROXY_HTTP2HTTPS | `false` | enable http->https proxy for images | 1.5.0 |
+</details>
+
 ##### Required parameters
 
-Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
+Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:
 
 1. `SECRET` - secret key, can be any long and hard-to-guess string.
-1. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
-2. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
+2. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.remark42.com`
+3. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
 
 The minimal `docker-compose.yml` has to include all required parameters:
 
@@ -81,19 +238,23 @@ version: '2'
 
 services:
     remark42:
-        image: umputun/remark42:master
+        image: umputun/remark42:latest
         restart: always
         container_name: "remark42"
         environment:
             - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
             - SECRET=abcd-123456-xyz-$%^&           # secret key
             - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
             - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
-            - USER=1001                             # UID on the host machine, i.e `id -u`
         volumes:
-            - ./var:/srv/var                        # persistent volume to store all remark42 data 
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
 ```
 
+#### Quick installation test
+
+To verify if remark has been properly installed, check a demo page at `${REMARK_URL}/web` URL. Make sure to include `remark` site id to `${SITE}` list.
+
 #### Register oauth2 providers
 
 Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
@@ -133,6 +294,22 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
 1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
 
+#### Microsoft Auth Provider
+
+1.  Register a new application [using the Azure portal](https://docs.microsoft.com/en-us/graph/auth-register-app-v2).
+2.  Under **"Authentication/Platform configurations/Web"** enter the correct url constructed as domain + `/auth/microsoft/callback`. i.e. `https://example.mysite.com/auth/microsoft/callback`
+3.  In "Overview" take note of the **Application (client) ID**
+4.  Choose the new project from the top right project dropdown (only if another project is selected)
+5.  Select "Certificates & secrets" and click on "+ New Client Secret".
+
+##### Twitter Auth Provider
+
+1.	Create a new twitter application https://developer.twitter.com/en/apps
+1.	Fill **App name**, **Description** and **URL** of your site
+1.	In the field **Callback URLs** enter the correct url of your callback handler e.g.  domain + `/auth/twitter/callback`
+1.	Under **Key and tokens** take note of the **Consumer API Key** and **Consumer API Secret key**. Those will be used as `AUTH_TWITTER_CID` and
+ `AUTH_TWITTER_CSEC`
+
 ##### Yandex Auth Provider
 
 1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
@@ -146,11 +323,29 @@ _instructions for google oauth2 setup borrowed from [oauth2_proxy](https://githu
 
 For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
 
+##### Anonymous Auth Provider
+
+Optionally, anonymous access can be turned on. In this case an extra `anonymous` provider will allow logins without any social login with any name satisfying 2 conditions:
+
+- name should be at least 3 characters long
+- name has to start from the letter and contains letters, numbers, underscores and spaces only.
+
+### Importing comments
+
+Remark supports importing comments from Disqus, WordPress or native backup format.
+All imported comments has `Imported` field set to `true`.
+
 #### Initial import from Disqus
 
 1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
 2.  Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Run import command - `docker-compose exec remark42 /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
+3.  Run import command - `docker exec -it remark42 import -p disqus -f /srv/var/{disqus-export-name}.xml -s {your site id}`
+
+#### Initial import from WordPress
+
+1. Install WordPress [plugin](https://wordpress.org/plugins/wp-exporter/) to export comments and follow it instructions. The plugin should produce a xml-based file with site content including comments.
+2. Move this file to your remark42 host within `./var`
+3. Run import command - `docker exec -it remark42 import -p wordpress -f {wordpress-export-name}.xml -s {your site id}`
 
 #### Backup and restore
 
@@ -159,109 +354,164 @@ Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./
 
 For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
 
-`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}`
-
-##### Schema migration
-
-One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema 
-and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance.
-
-`docker-compose exec remark42 /srv/migrate-data.sh {your site id}`
+`docker exec -it remark42 restore -f {backup-filename.gz} -s {your site id}`
 
 ##### Manual backup
 
-In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz`
+In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz` by default.
 
-`docker-compose exec remark42 /srv/create-backup.sh {your site id}`
+`docker exec -it remark42 backup -s {your site id}`
+
+##### Restore from backup
+
+Restore will clean all comments first and then will processed with complete import from a given file.
+
+`docker exec -it remark42 restore -f {backup file name} -s {your site id}`
 
 ##### Backup format
 
 Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
-unmarshaled from `Comment` struct (see below). 
+unmarshaled from `Comment` struct (see below).
 
 #### Admin users
 
-Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line. 
+Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line.
 
 ```
     environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
+        - ADMIN_SHARED_ID=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
 ```
 
-To get user id just login and click on your username or any other user you want to promote to admins. 
+To get user id just login and click on your username or any other user you want to promote to admins.
 It will expand login info and show full user ID.
 
+#### Docker parameters
 
-### Frontend
+Two parameters allow customizing Docker container on the system level:
 
-Frontend part is building automatically along with backend if you use `docker-compose`.
+- `APP_UID` - sets UID to run remark42 application in container (default=1001)
+- `TIME_ZONE` - sets time zone of remark42 container (default=America/Chicago)
 
-For manual building:
+_see [umputun/baseimage](https://github.com/umputun/baseimage) for more details_
 
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* run `npm install` inside `./web`;
-* run `npm run build` there;
-* result files will be saved in `./web/public`.
+example of `docker-compose.yml`:
 
-For development mode use `npm start` instead of `npm run build`.
-In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
+```yaml
+version: '2'
 
-URLs for development:
+services:
+    remark42:
+        image: umputun/remark42:latest
+        restart: always
+        container_name: "remark42"
+        environment:
+            - APP_UID=2000                          # runs remark42 app with non-default UID
+            - TIME_ZONE=GTC                         # sets container time to UTC
 
-* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
-* `localhost:8080/dev.html` — page with embedded script from local folder;
-* `localhost:8080/last-comments.html` — page with embedded script for last comments;
-* `localhost:8080/counter.html` — page with embedded script for counter with examples.
+            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
+            - SITE=YOUR_SITE_ID                     # site ID, same as used for `site_id`, see "Setup on your website"
+            - SECRET=abcd-123456-xyz-$%^&           # secret key
+            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
+            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
+        volumes:
+            - ./var:/srv/var                        # persistent volume to store all remark42 data
+```
 
-#### Usage
 
-##### Comments
+### Setup on your website
 
-It's a main widget which renders list of comments. 
+#### Comments
+
+It's a main widget which renders list of comments.
 
 Add this snippet to the bottom of web page:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
-    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used 
+    components: ['embed'], // optional param; which components to load. default to ["embed"]
+                           // to load all components define components as ['embed', 'last-comments', 'counter']
+                           // available component are:
+                           //     - 'embed': basic comments widget
+                           //     - 'last-comments': last comments widget, see `Last Comments` section below
+                           //     - 'counter': counter widget, see `Counter` section below
+    url: 'PAGE_URL', // optional param; if it isn't defined
+                     // `window.location.origin + window.location.pathname` will be used,
+                     //
+                     // Note that if you use query parameters as significant part of url
+                     // (the one that actually changes content on page)
+                     // you will have to configure url manually to keep query params, as
+                     // `window.location.origin + window.location.pathname` doesn't contain query params and
+                     // hash. For example default url for `https://example/com/example-post?id=1#hash`
+                     // would be `https://example/com/example-post`.
+                     //
+                     // The problem with query params is that they often contain useless params added by
+                     // various trackers (utm params) and doesn't have defined order, so Remark treats differently
+                     // all this examples:
+                     // https://example.com/?postid=1&date=2007-02-11
+                     // https://example.com/?date=2007-02-11&postid=1
+                     // https://example.com/?date=2007-02-11&postid=1&utm_source=google
+                     //
+                     // If you deal with query parameters make sure you pass only significant part of it
+                     // in well defined order
+    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
+    theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
+    page_title: 'Moving to Remark42', // optional param; if it isn't defined `document.title` will be used
+    locale: 'en', // set up locale and language, if it isn't defined default value ('en') will be used
+    show_email_subscription: false // optional param; by default it is `true` and you can see email subscription feature
+                                   // in interface when enable it from backend side
+                                   // if you set this param in `false` you will get notifications email notifications as admin
+                                   // but your users won't have interface for subscription
   };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
 </script>
+<script>!function(e,n){for(var o=0;o<e.length;o++){var r=n.createElement("script"),c=".js",d=n.head||n.body;"noModule"in r?(r.type="module",c=".mjs"):r.async=!0,r.defer=!0,r.src=remark_config.host+"/web/"+e[o]+c,d.appendChild(r)}}(remark_config.components||["embed"],document);</script>
 ```
 
 And then add this node in the place where you want to see Remark42 widget:
 
 ```html
 <div id="remark42"></div>
-``` 
+```
 
 After that widget will be rendered inside this node.
 
-##### Last comments
+If you want to set this up on a Single Page App, see [appropriate doc page](https://remark42.com/docs/latest/spa/).
+
+##### Themes
+
+Right now Remark has two themes: light and dark.
+You can pick one using configuration object,
+but there is also a possibility to switch between themes in runtime.
+For this purpose Remark adds to `window` object named `REMARK42`,
+which contains function `changeTheme`.
+Just call this function and pass a name of the theme that you want to turn on:
+
+```js
+window.REMARK42.changeTheme('light');
+```
+
+##### Locales
+
+Right now Remark is translated to en, ru (partially), de, and fi languages.
+You can pick one using [configuration object](#setup-on-your-website).
+
+Do you want translate remark42 to other locale? Please see [this documentation](https://github.com/umputun/remark42/blob/master/docs/translation.md) for details.
+
+#### Last comments
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
+    site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
 </script>
 ```
 
@@ -273,23 +523,19 @@ And then add this node in the place where you want to see last comments widget:
 
 `data-max` sets the max amount of comments (default: `15`).
 
-##### Counter
+#### Counter
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
+    site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
 </script>
 ```
 
@@ -299,17 +545,103 @@ And then add a node like this in the place where you want to see a number of com
 <span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
 ```
 
-You can use as many nodes like this as you need to. 
-The script will found all them by the class `remark__counter`, 
+You can use as many nodes like this as you need to.
+The script will found all them by the class `remark__counter`,
 and it will use `data-url` attribute to define the page with comments.
 
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
+Also script can use `url` property from `remark_config` object, or `window.location.origin + window.location.pathname` if nothing else is defined.
+
+## Build from the source
+
+- to build Docker container - `make docker`. This command will produce container `umputun/remark42`.
+- to build a single binary for direct execution - `make OS=<linux|windows|darwin> ARCH=<amd64|386>`. This step will produce executable
+ `remark42` file with everything embedded.
+
+## Development
+
+You can use fully functional local version to develop and test both frontend & backend. It requires at least 2GB RAM or swap enabled
+
+To bring it up run:
+
+```bash
+# if you mainly work on backend
+cp compose-dev-backend.yml compose-private.yml
+# if you mainly work on frontend
+cp compose-dev-frontend.yml compose-private.yml
+# now, edit / debug `compose-private.yml` to your heart's content.
+
+# build and run
+docker-compose -f compose-private.yml build
+docker-compose -f compose-private.yml up
+```
+
+It starts Remark42 on `127.0.0.1:8080` and adds local OAuth2 provider “Dev”.
+To access UI demo page go to `127.0.0.1:8080/web`.
+By default, you would be logged in as `dev_user` which defined as admin.
+You can tweak any of [supported parameters](#Parameters) in corresponded yml file.
+
+Backend Docker Compose config by default skips running frontend related tests.
+Frontend Docker Compose config by default skips running backend related tests and sets `NODE_ENV=development` for frontend build.
+
+### Backend development
+
+In order to run backend locally (development mode, without Docker) you have to have the latest stable `go` toolchain [installed](https://golang.org/doc/install).
+
+To run backend - `cd backend; go run app/main.go server --dbg --secret=12345 --url=http://127.0.0.1:8080 --admin-passwd=password --site=remark`
+It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
+`HTTP http://admin:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
+
+### Frontend development
+
+#### Developer guide
+
+Frontend guide can be found here: [./frontend/README.md](./frontend/README.md)
+
+#### Build
+You should have at least 2GB RAM or swap enabled for building
+
+* install [Node.js 12.11](https://nodejs.org/en/) or higher;
+* install [NPM 6.13.4](https://www.npmjs.com/package/npm);
+* run `npm install` inside `./frontend`;
+* run `npm run build` there;
+* result files will be saved in `./frontend/public`.
+
+**Note** Running `npm install` will set up precommit hooks into your git repository.
+It used to reformat your frontend code using `prettier` and lint with `eslint` and `stylelint` before every commit.
+
+#### Devserver
+
+For local development mode with Hot Reloading use `npm start` instead of `npm run build`.
+In this case `webpack` will serve files using `webpack-dev-server` on `localhost:9000`.
+By visiting `127.0.0.1:9000/web` you will get a page with main comments widget
+communicating with demo server backend running on `https://demo.remark42.com`.
+But you will not be able to login with any oauth providers due to security reasons.
+
+You can attach to locally running backend by providing `REMARK_URL` environment variable.
+```sh
+npx cross-env REMARK_URL=http://127.0.0.1:8080 npm start
+```
+
+**Note** If you want to redefine env variables such as `PORT` on your local instance you can add `.env` file
+to `./frontend` folder and rewrite variables as you wish. For such functional we use `dotenv`
+
+The best way for start local developer environment:
+```sh
+cp compose-dev-frontend.yml compose-private-frontend.yml
+docker-compose -f compose-private-frontend.yml up --build
+cd frontend
+npm run dev
+```
+
+Developer build running by `webpack-dev-server` supports devtools for [React](https://github.com/facebook/react-devtools) and
+[Redux](https://github.com/zalmoxisus/redux-devtools-extension).
+
 
 ## API
 
 ### Authorization
 
-* `GET /auth/{provider}/login?from=http://url&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
+* `GET /auth/{provider}/login?from=http://url&site=site_id&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
 * `GET /auth/logout` - logout
 
 ```go
@@ -338,16 +670,24 @@ type Comment struct {
     User      User            `json:"user"`    // user info, read only
     Locator   Locator         `json:"locator"` // post locator
     Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
+    Vote      int             `json:"vote"`    // vote for the current user, -1/1/0.
+    Controversy float64       `json:"controversy,omitempty"` // comment controversy, read only
     Timestamp time.Time       `json:"time"`    // time stamp, read only
+    Edit      *Edit           `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
     Pin       bool            `json:"pin"`     // pinned status, read only
     Delete    bool            `json:"delete"`  // delete status, read only
+    PostTitle string          `json:"title"`   // post title
 }
 
 type Locator struct {
     SiteID string `json:"site"`     // site id
     URL    string `json:"url"`      // post url
 }
+
+type Edit struct {
+  Timestamp time.Time `json:"time" bson:"time"`
+  Summary   string    `json:"summary"`
+}
 ```
 
 * `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
@@ -371,18 +711,17 @@ type Node struct {
 
 Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
 
-* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in 5min since creation
+* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in `EDIT_TIME` minutes since creation.  Body is `EditRequest` json
 
-```json
-  Content-Type: application/json
-
-  {
-    "text": "edit comment blah http://radio-t.com 12345",
-    "summary": "fix blah"
-  }
+```go
+   type EditRequest struct {
+     Text    string `json:"text"`    // updated text
+     Summary string `json:"summary"` // optional, summary of the edit
+     Delete  bool   `json:"delete"`  // delete flag
+   }{}
 ```
 
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
+* `GET /api/v1/last/{max}?site=site-id&since=ts-msec` - get up to `{max}` last comments, `since` (epoch time, milliseconds) is optional
 * `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
 * `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
   ```go
@@ -392,6 +731,7 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
   }{}
   ```
 * `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
+* `POST /api/v1/count?site=siteID` - get number of comments for posts from post body (list of post IDs)
 * `GET /api/v1/list?site=site-id&limit=5&skip=2` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
   ```go
   type PostInfo struct {
@@ -409,73 +749,145 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
 
   ```go
-  type config struct {
-      Version       string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
-      Admins        []string `json:"admins"`
-      Auth          []string `json:"auth_providers"`
-      LowScore      int      `json:"low_score"`
-      CriticalScore int      `json:"critical_score"`
+  type Config struct {
+        Version        string   `json:"version"`
+        EditDuration   int      `json:"edit_duration"`
+        MaxCommentSize int      `json:"max_comment_size"`
+        Admins         []string `json:"admins"`
+        AdminEmail     string   `json:"admin_email"`
+        Auth           []string `json:"auth_providers"`
+        LowScore       int      `json:"low_score"`
+        CriticalScore  int      `json:"critical_score"`
+        PositiveScore  bool     `json:"positive_score"`
+        ReadOnlyAge    int      `json:"readonly_age"`
+        MaxImageSize   int      `json:"max_image_size"`
+        EmojiEnabled   bool     `json:"emoji_enabled"`
   }
-  ``` 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
-  
+  ```
+
+* `GET /api/v1/info?site=site-idd&url=post-url` - returns `PostInfo` for site and url
+
+### Streaming API
+
+Streaming API provide server-sent events for post updates as well as site update
+
+* `GET /api/v1/stream/info?site=site-idd&url=post-url&since=unix_ts_msec` - returns stream (`event: info`) with `PostInfo` records for the site and url. `since` is optional
+* `GET /api/v1/stream/last?site=site-id&since=unix_ts_msec` - returns updates stream (`event: last`) with comments for the site, `since` is optional
+
+<details><summary>response example</summary>
+
+```
+data: {"url":"https://radio-t.com/blah1","count":2,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.142872-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":3,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.157709-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":4,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.172991-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":5,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.188429-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":6,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.204742-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":7,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.220692-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":8,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.23817-05:00"}
+
+event: info
+data: {"url":"https://radio-t.com/blah1","count":9,"first_time":"2019-06-18T12:53:48.125686-05:00","last_time":"2019-06-18T12:53:48.254669-05:00"}
+```
+
+</details>
+
 ### RSS feeds
-  
+
 * `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
+* `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
+
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+
+### Email subscription
+
+* `GET /api/v1/email?site=site-id` - get user's email, _auth required_
+* `POST /api/v1/email/subscribe?site=site-id&address=user@example.org` -  makes confirmation token and sends it to user over email, _auth required_
+
+  Trying to subscribe same email second time will return response code `409 Conflict` and explaining error message.
+* `POST /api/v1/email/confirm?site=site-id&tkn=token` - uses provided token parameter to set email for the user, _auth required_
+
+  Setting email subscribe user for all first-level replies to his messages.
+* `DELETE /api/v1/email?site=siteID` - removes user's email, _auth required_
 
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
-* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1` - block or unblock user.
-* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids.
+* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1&ttl=7d` - block or unblock user with optional ttl (default=permanent)
+* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids
   ```go
   type BlockedUser struct {
       ID        string    `json:"id"`
       Name      string    `json:"name"`
-      Timestamp time.Time `json:"time"`
+      Until     time.Time `json:"time"`
   }
   ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup.
+* `GET /api/v1/admin/export?site=site-id&mode=[stream|file]` - export all comments to json stream or gz file.
+* `POST /api/v1/admin/import?site=site-id` - import comments from the backup, uses post body.
+* `POST /api/v1/admin/import/form?site=site-id` - import comments from the backup, user post form.
+* `POST /api/v1/admin/remap?site=site-id` - remap comments to different URLs. Expect list of "from-url new-url" pairs separated by \n.
+From-url and new-url parts separated by space. If urls end with asterisk (*) it means matching by prefix. Remap procedure based on
+export/import chain so make backup first.
+    ```
+    http://oldsite.com* https://newsite.com*
+    http://oldsite.com/from-old-page/1 https://newsite.com/to-new-page/1
+    ```
+* `GET /api/v1/admin/wait?site=site-id` - wait for completion for any async migration ops (import or remap).
 * `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
-* `DELETE /api/v1/admin/user/{userid}?site=site-id&block=1` - delete all user's comments.
+* `GET /api/v1/admin/user/{userid}?site=site-id` - get user's info.
+* `DELETE /api/v1/admin/user/{userid}?site=site-id` - delete all user's comments.
 * `PUT /api/v1/admin/readonly?site=site-id&url=post-url&ro=1` - set read-only status
 * `PUT /api/v1/admin/verify/{userid}?site=site-id&verified=1` - set verified status
 * `GET /api/v1/admin/deleteme?token=token` - process deleteme user's request
 
 _all admin calls require auth and admin privilege_
 
-
-## Privacy 
+## Privacy
 
 * Remark42 is trying to be very sensitive to any private or semi-private information.
-* Authentication requesting the lowest (minimal) possible scope from providers. All extra information returned by them dropped immediately and not stored in any form.
-* Generally remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
+* Authentication requesting the minimal possible scope from authentication providers. All extra information returned by them dropped immediately and not stored in any form.
+* Generally, remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
 * There is no tracking of any sort.
-* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing Cross-Site Request Forgery
+* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing CSRF.
 * There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
 * There are no third-party analytic services involved.
 * User can request all information remark42 knows about and export to gz file.
-* Supported complete cleanup of all information related to user activity on demand.
-* Cookie lifespan can be restricted to session-only. 
+* Supported complete cleanup of all information related to user's activity.
+* Cookie lifespan can be restricted to session-only.
 * All potentially sensitive data stored by remark42 hashed and encrypted.
 
-
 ## Technical details
 
-* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `BOLTDB_PATH`
+* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `STORE_BOLT_PATH`
 * Each site stored in a separate boltbd file.
-* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_STORE` should be transferred.
+* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_FS_PATH` should be transferred. Optionally, boltdb can be used to store avatars as well.
 * Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
-* Authentication implemented with [jwt](https://github.com/dgrijalva/jwt-go) stored in a cookie. It uses HttpOnly, secure cookies.
-* All heavy REST calls cached internally in LRU cache limited by `MAX_CACHE_ITEMS` and `MAX_CACHE_SIZE`.
+* Authentication implemented with [go-pkgz/auth](https://github.com/go-pkgz/auth) stored in a cookie. It uses HttpOnly, secure cookies.
+* All heavy REST calls cached internally in LRU cache limited by `CACHE_MAX_ITEMS` and `CACHE_MAX_SIZE` with [go-pkgz/rest](https://github.com/go-pkgz/rest)
 * User's activity throttled globally (up to 1000 simultaneous requests) and limited locally (per user, usually up to 10 req/sec)
 * Request timeout set to 60sec
-* Development mode (`--dev-password` set) allows to test remark42 without social login and with admin privileges. Adds basic-auth for username: `dev`, password: `${DEV_PASSWD}`. **should not be used in production deployment**
-* User can vote for the comment multiple times but only to change his/her vote. Double-voting not allowed.
-* User can edit comments in 5 mins window after creation.
+* Admin authentication (`--admin-password` set) allows to hit remark42 API without social login and with admin privileges. Adds basic-auth for username: `admin`, password: `${ADMIN_PASSWD}`.
+* User can vote for the comment multiple times but only to change the vote. Double-voting not allowed.
+* User can edit comments in 5 mins (configurable) window after creation.
 * User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
-* All avatars cached locally to prevent rate limiters from google/github/facebook/yandex.
+* All avatars resized and cached locally to prevent rate limiters from oauth providers, part of [go-pkgz/auth](https://github.com/go-pkgz/auth) functionality.
+* Images can be proxied (`IMAGE_PROXY_HTTP2HTTPS=true`) to prevent mixed http/https.
+* All images can be proxied and saved (`IMAGE_PROXY_CACHE_EXTERNAL=true`) instead of serving from original location. Beware, images which are posted with this parameter enabled will be served from proxy even after it will be disabled.
 * Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.

SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| current | :white_check_mark: 
+| 1.6.x   | :white_check_mark: |
+| <1.5.x | :x:                 |
+
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to umputun@gmail.com. You will receive a response from us within 48 hours.
+If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.
+

app/main.go

@@ -1,398 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/hashicorp/logutils"
-	"github.com/jessevdk/go-flags"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/api"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Opts with command line flags and env
-// nolint:maligned
-type Opts struct {
-	SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
-	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
-
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-
-	Sites          []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	Admins         []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
-	AdminEmail     string   `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
-	DevPasswd      string   `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
-	BackupLocation string   `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles int      `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy     bool     `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize int      `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	LowScore       int      `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore  int      `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	ReadOnlyAge    int      `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
-	Port           int      `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot        string   `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	Dbg            bool     `long:"dbg" env:"DEBUG" description:"debug mode"`
-
-	Auth struct {
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
-}
-
-// AuthGroup defines options group for auth params
-type AuthGroup struct {
-	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
-	CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
-}
-
-// StoreGroup defines options group for store params
-type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
-	Bolt struct {
-		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
-		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
-	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
-}
-
-// AvatarGroup defines options group for avatar params
-type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
-	FS   struct {
-		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
-	} `group:"fs" namespace:"fs" env-namespace:"FS"`
-	RszLmt int `long:"rsz-lmt" env:"RSZ_LMT" default:"0" description:"max image size for resizing avatars on save"`
-}
-
-// CacheGroup defines options group for cache params
-type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
-	Max  struct {
-		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
-		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
-		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
-	} `group:"max" namespace:"max" env-namespace:"MAX"`
-}
-
-var revision = "unknown"
-
-// Application holds all active objects
-type Application struct {
-	Opts
-	restSrv     *api.Rest
-	migratorSrv *api.Migrator
-	exporter    migrator.Exporter
-	terminated  chan struct{}
-}
-
-func main() {
-	fmt.Printf("remark %s\n", revision)
-
-	var opts Opts
-	p := flags.NewParser(&opts, flags.Default)
-	if _, e := p.ParseArgs(os.Args[1:]); e != nil {
-		os.Exit(1)
-	}
-
-	setupLog(opts.Dbg)
-	log.Print("[INFO] started remark")
-	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() { // catch signal and invoke graceful termination
-		stop := make(chan os.Signal, 1)
-		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
-		<-stop
-		log.Print("[WARN] interrupt signal")
-		cancel()
-	}()
-
-	app, err := New(opts)
-	if err != nil {
-		log.Fatalf("[ERROR] failed to setup application, %+v", err)
-	}
-	err = app.Run(ctx)
-	log.Printf("[INFO] remark terminated %s", err)
-}
-
-// New prepares application and return it with all active parts
-// doesn't start anything
-func New(opts Opts) (*Application, error) {
-
-	if err := makeDirs(opts.BackupLocation); err != nil {
-		return nil, err
-	}
-
-	if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
-		return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
-	}
-
-	boltStore, err := makeDataStore(opts.Store, opts.Sites)
-	if err != nil {
-		return nil, err
-	}
-	dataService := &service.DataStore{
-		Interface:      boltStore,
-		EditDuration:   5 * time.Minute,
-		Secret:         opts.SecretKey,
-		MaxCommentSize: opts.MaxCommentSize,
-	}
-
-	loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
-		cache.MaxKeys(opts.Cache.Max.Items), cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
-	if err != nil {
-		return nil, err
-	}
-
-	jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour)
-
-	avatarStore, err := makeAvatarStore(opts.Avatar)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make avatar store")
-	}
-	avatarProxy := &proxy.Avatar{
-		Store:     avatarStore,
-		RoutePath: "/api/v1/avatar",
-		RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
-	}
-
-	exporter := &migrator.Remark{DataStore: dataService}
-
-	migr := &api.Migrator{
-		Version:        revision,
-		Cache:          loadingCache,
-		NativeImporter: &migrator.Remark{DataStore: dataService},
-		DisqusImporter: &migrator.Disqus{DataStore: dataService},
-		NativeExported: &migrator.Remark{DataStore: dataService},
-		SecretKey:      opts.SecretKey,
-	}
-
-	srv := &api.Rest{
-		Version:     revision,
-		DataService: dataService,
-		Exporter:    exporter,
-		WebRoot:     opts.WebRoot,
-		RemarkURL:   opts.RemarkURL,
-		ImageProxy:  &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
-		AvatarProxy: avatarProxy,
-		ReadOnlyAge: opts.ReadOnlyAge,
-		Authenticator: auth.Authenticator{
-			JWTService: jwtService,
-			Admins:     opts.Admins,
-			AdminEmail: opts.AdminEmail,
-			Providers:  makeAuthProviders(jwtService, avatarProxy, dataService, opts),
-			DevPasswd:  opts.DevPasswd,
-		},
-		Cache: loadingCache,
-	}
-
-	// no admin email, use admin@domain
-	if srv.Authenticator.AdminEmail == "" {
-		if u, err := url.Parse(opts.RemarkURL); err == nil {
-			srv.Authenticator.AdminEmail = "admin@" + u.Host
-		}
-	}
-
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
-	tch := make(chan struct{})
-	return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, Opts: opts, terminated: tch}, nil
-}
-
-// Run all application objects
-func (a *Application) Run(ctx context.Context) error {
-	if a.DevPasswd != "" {
-		log.Printf("[WARN] running in dev mode")
-	}
-
-	go func() {
-		// shutdown on context cancellation
-		<-ctx.Done()
-		a.restSrv.Shutdown()
-		a.migratorSrv.Shutdown()
-	}()
-	a.activateBackup(ctx) // runs in goroutine for each site
-	go a.migratorSrv.Run(a.Port + 1)
-	a.restSrv.Run(a.Port)
-	close(a.terminated)
-	return nil
-}
-
-// Wait for application completion (termination)
-func (a *Application) Wait() {
-	<-a.terminated
-}
-
-// activateBackup runs background backups for each site
-func (a *Application) activateBackup(ctx context.Context) {
-	for _, siteID := range a.Sites {
-		backup := migrator.AutoBackup{
-			Exporter:       a.exporter,
-			BackupLocation: a.BackupLocation,
-			SiteID:         siteID,
-			KeepMax:        a.MaxBackupFiles,
-			Duration:       24 * time.Hour,
-		}
-		go backup.Do(ctx)
-	}
-}
-
-// makeDataStore creates store for all sites
-func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
-	switch group.Type {
-	case "bolt":
-		if err = makeDirs(group.Bolt.Path); err != nil {
-			return nil, err
-		}
-		sites := []engine.BoltSite{}
-		for _, site := range siteNames {
-			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
-		}
-		result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
-	default:
-		return nil, errors.Errorf("unsupported store type %s", group.Type)
-	}
-
-	return result, errors.Wrap(err, "can't initialize data store")
-}
-
-func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
-	switch group.Type {
-	case "fs":
-		if err = makeDirs(group.FS.Path); err != nil {
-			return nil, err
-		}
-		return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
-	}
-	return nil, errors.Errorf("unsupported avatart store type %s", group.Type)
-}
-
-// mkdir -p for all dirs
-func makeDirs(dirs ...string) error {
-
-	// exists returns whether the given file or directory exists or not
-	exists := func(path string) (bool, error) {
-		_, err := os.Stat(path)
-		if err == nil {
-			return true, nil
-		}
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return true, err
-	}
-
-	for _, dir := range dirs {
-		ex, err := exists(dir)
-		if err != nil {
-			return errors.Wrapf(err, "can't check directory status for %s", dir)
-		}
-		if !ex {
-			if e := os.MkdirAll(dir, 0700); e != nil {
-				return errors.Wrapf(err, "can't make directory %s", dir)
-			}
-		}
-	}
-	return nil
-}
-
-func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
-
-	makeParams := func(cid, secret string) auth.Params {
-		return auth.Params{
-			JwtService:   jwtService,
-			AvatarProxy:  avatarProxy,
-			RemarkURL:    opts.RemarkURL,
-			Cid:          cid,
-			Csecret:      secret,
-			Admins:       opts.Admins,
-			SecretKey:    opts.SecretKey,
-			IsVerifiedFn: ds.IsVerifiedFn(),
-		}
-	}
-
-	providers := []auth.Provider{}
-	if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
-		providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
-	}
-	if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
-		providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
-	}
-	if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
-		providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
-	}
-	if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
-		providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
-	}
-	if len(providers) == 0 {
-		log.Printf("[WARN] no auth providers defined")
-	}
-	return providers
-}
-
-// post-flush callback invoked by cache after each flush in async way
-func postFlushFn(sites []string, port int) func() {
-
-	return func() {
-		// list of heavy urls for pre-heating on cache change
-		urls := []string{
-			"http://localhost:%d/api/v1/list?site=%s",
-			"http://localhost:%d/api/v1/last/50?site=%s",
-		}
-
-		for _, site := range sites {
-			for _, u := range urls {
-				resp, err := http.Get(fmt.Sprintf(u, port, site))
-				if err != nil {
-					log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
-					return
-				}
-				if err = resp.Body.Close(); err != nil {
-					log.Printf("[WARN] failed to close response body, %s", err)
-				}
-			}
-		}
-	}
-}
-
-func resetEnv(envs ...string) {
-	for _, env := range envs {
-		if err := os.Unsetenv(env); err != nil {
-			log.Printf("[WARN] can't unset env %s, %s", env, err)
-		}
-	}
-}
-
-func setupLog(dbg bool) {
-	filter := &logutils.LevelFilter{
-		Levels:   []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},
-		MinLevel: logutils.LogLevel("INFO"),
-		Writer:   os.Stdout,
-	}
-
-	log.SetFlags(log.Ldate | log.Ltime)
-
-	if dbg {
-		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
-		filter.MinLevel = logutils.LogLevel("DEBUG")
-	}
-	log.SetOutput(filter)
-}

app/main_test.go

@@ -1,135 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	flags "github.com/jessevdk/go-flags"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestApplication(t *testing.T) {
-	app, ctx := prepApp(t, 18080, 500*time.Millisecond)
-	go func() { _ = app.Run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
-
-	// add comment
-	resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
-		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
-	require.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	body, _ = ioutil.ReadAll(resp.Body)
-	t.Log(string(body))
-
-	assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
-
-	app.Wait()
-}
-
-func TestApplicationFailed(t *testing.T) {
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-
-	// RO bolt location
-	_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/dev/null"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
-		"open /dev/null/remark.db: not a directory")
-	t.Log(err)
-
-	// RO backup location
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
-		"--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
-	t.Log(err)
-
-	// invalid url
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
-	t.Log(err)
-
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.type=mongo"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "unsupported store type mongo")
-	t.Log(err)
-}
-
-func TestApplicationShutdown(t *testing.T) {
-	app, ctx := prepApp(t, 18090, 500*time.Millisecond)
-	st := time.Now()
-	err := app.Run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-	app.Wait()
-}
-
-func TestApplicationMainSignal(t *testing.T) {
-	os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
-		"--port=18100", "--url=https://demo.remark42.com"}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
-	}()
-	st := time.Now()
-	main()
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-}
-
-func prepApp(t *testing.T, port int, duration time.Duration) (*Application, context.Context) {
-	// prepare options
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
-	require.Nil(t, err)
-	opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
-	opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", port)
-	opts.Store.Bolt.Timeout = 10 * time.Second
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
-	opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
-	opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
-	opts.Port = port
-
-	os.Remove(opts.Store.Bolt.Path + "/remark.db")
-
-	// create app
-	app, err := New(opts)
-	require.Nil(t, err)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	return app, ctx
-}

app/migrator/migrator_test.go

@@ -1,93 +0,0 @@
-package migrator
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store/service"
-
-	"github.com/umputun/remark/app/store/engine"
-)
-
-func TestMigrator_ImportDisqus(t *testing.T) {
-	defer func() {
-		os.Remove("/tmp/remark-test.db")
-		os.Remove("/tmp/disqus-test.xml")
-	}()
-
-	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600)
-	require.Nil(t, err)
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-	size, err := ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.xml",
-		SiteID:    "test",
-		Provider:  "disqus",
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 3, size)
-
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
-}
-
-func TestMigrator_ImportRemark(t *testing.T) {
-	defer func() {
-		os.Remove("/tmp/remark-test.db")
-		os.Remove("/tmp/disqus-test.r42")
-	}()
-
-	data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
-		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-
-	size, err := ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.r42",
-		SiteID:    "radio-t",
-		Provider:  "native",
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	last, err := dataStore.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(last), "2 comments imported")
-}
-
-func TestMigrator_ImportFailed(t *testing.T) {
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-	_, err = ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.xml",
-		SiteID:    "test",
-		Provider:  "bad",
-	})
-
-	assert.EqualError(t, err, "unsupported import provider bad")
-
-	_, err = ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test-bad.xml",
-		SiteID:    "test",
-		Provider:  "native",
-	})
-	assert.EqualError(t, err, "can't open import file /tmp/disqus-test-bad.xml: open /tmp/disqus-test-bad.xml: no such file or directory")
-}

app/migrator/remark.go

@@ -1,95 +0,0 @@
-package migrator
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"io"
-	"log"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/store"
-)
-
-// Remark implements exporter and importer for internal store format
-type Remark struct {
-	DataStore Store
-}
-
-// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
-func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
-	topics, err := r.DataStore.List(siteID, 0, 0)
-	if err != nil {
-		return 0, err
-	}
-	log.Printf("[DEBUG] exporting %d topics", len(topics))
-
-	commentsCount := 0
-	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
-		topic := topics[i]
-		comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
-			return commentsCount, err
-		}
-
-		for _, comment := range comments {
-
-			buf := &bytes.Buffer{}
-			enc := json.NewEncoder(buf)
-			enc.SetEscapeHTML(false)
-
-			if err := enc.Encode(comment); err != nil {
-				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
-			}
-			data := buf.Bytes()
-			if _, err := w.Write(data); err != nil {
-				return commentsCount, errors.Wrap(err, "can't write comment data")
-			}
-			commentsCount++
-		}
-	}
-	log.Printf("[DEBUG] exported %d comments", commentsCount)
-	return commentsCount, nil
-}
-
-// Import comments from json strings produced by Remark.Export
-func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
-
-	if err := r.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
-	}
-
-	failed := 0
-	total, comments := 0, 0
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		rec := scanner.Bytes()
-		if len(rec) < 3 {
-			continue
-		}
-		total++
-		comment := store.Comment{}
-		if err := json.Unmarshal(rec, &comment); err != nil {
-			failed++
-			log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
-			continue
-		}
-		if _, err := r.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
-	}
-	if scanner.Err() != nil {
-		return comments, errors.Wrap(scanner.Err(), "error in scan")
-	}
-	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
-	}
-	log.Printf("[INFO] imported %d comments from %d records", comments, total)
-	return comments, nil
-}

app/migrator/remark_test.go

@@ -1,112 +0,0 @@
-package migrator
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestRemark_Export(t *testing.T) {
-	b := prep(t)
-	r := Remark{DataStore: b}
-
-	buf := &bytes.Buffer{}
-	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	c1, err := buf.ReadString('\n')
-	assert.Nil(t, err)
-	log.Print(c1)
-	exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-	assert.Equal(t, exp, c1)
-}
-
-func TestRemark_Import(t *testing.T) {
-	r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	buf.WriteString(r1)
-	buf.WriteString(r2)
-	buf.WriteString("{}")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	size, err := r.Import(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[0].ID)
-	assert.Equal(t, "afbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ParentID)
-}
-
-func TestRemark_ImportManyWithError(t *testing.T) {
-	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < 1200; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
-	}
-	buf.WriteString("bad1\n")
-	buf.WriteString("bad2\n")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	n, err := r.Import(buf, "radio-t")
-	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-
-	b := &service.DataStore{Interface: boltStore}
-
-	comment := store.Comment{
-		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/rest/api/admin.go

@@ -1,245 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	"github.com/umputun/remark/app/rest/auth"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// admin provides router for all requests available for admin users only
-type admin struct {
-	dataService   *service.DataStore
-	exporter      migrator.Exporter
-	cache         cache.LoadingCache
-	authenticator auth.Authenticator
-	readOnlyAge   int
-}
-
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Get("/export", a.exportCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	return router
-}
-
-// DELETE /comment/{id}?site=siteID&url=post-url - removes comment
-func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	id := chi.URLParam(r, "id")
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	log.Printf("[INFO] delete comment %s", id)
-
-	err := a.dataService.Delete(locator, id, store.SoftDelete)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
-		return
-	}
-	a.cache.Flush(locator.SiteID, locator.URL)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"id": id, "locator": locator})
-}
-
-// DELETE /user/{userid}?site=side-id - delete all user comments for requested userid
-func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
-
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
-
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
-}
-
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
-// request made GET to allow direct click from the email sent by user
-func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
-
-	token := r.URL.Query().Get("token")
-
-	claims, err := a.authenticator.JWTService.Parse(token)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
-		return
-	}
-
-	log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID)
-
-	if err := a.dataService.DeleteUser(claims.SiteID, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
-		return
-	}
-	a.cache.Flush(claims.SiteID, claims.User.ID)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
-}
-
-// PUT /user/{userid}?site=side-id&block=1 - block or unblock user
-func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	blockStatus := r.URL.Query().Get("block") == "1"
-
-	if err := a.dataService.SetBlock(siteID, userID, blockStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
-}
-
-// GET /blocked?site=siteID - list blocked users
-func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
-		return
-	}
-	render.JSON(w, r, users)
-}
-
-// PUT /readonly?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
-func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	roStatus := r.URL.Query().Get("ro") == "1"
-
-	isRoByAge := func(info store.PostInfo) bool {
-		return a.readOnlyAge > 0 && !info.FirstTS.IsZero() &&
-			info.FirstTS.AddDate(0, 0, a.readOnlyAge).Before(time.Now())
-	}
-
-	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
-	if !roStatus {
-		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
-			return
-		}
-	}
-
-	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
-		return
-	}
-	a.cache.Flush(locator.SiteID)
-	render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus})
-}
-
-// PUT /verify?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
-func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	verifyStatus := r.URL.Query().Get("verified") == "1"
-
-	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus})
-}
-
-// PUT /pin/{id}?site=siteID&url=post-url&pin=1
-// mark/unmark comment as a special
-func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
-	commentID := chi.URLParam(r, "id")
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	pinStatus := r.URL.Query().Get("pin") == "1"
-
-	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
-		return
-	}
-	a.cache.Flush(locator.URL)
-	render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus})
-}
-
-// GET /export?site=site-id?mode=file|stream
-// exports all comments for siteID as json stream or gz file
-func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	var writer io.Writer = w
-	if r.URL.Query().Get("mode") == "file" {
-		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-		w.Header().Set("Content-Type", "application/gzip")
-		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
-		gzWriter := gzip.NewWriter(w)
-		defer func() {
-			if e := gzWriter.Close(); e != nil {
-				log.Printf("[WARN] can't close gzip writer, %s", e)
-			}
-		}()
-		writer = gzWriter
-	}
-
-	if _, err := a.exporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
-}
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin // make separate cache key for admins
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

app/rest/api/admin_test.go

@@ -1,541 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestAdmin_Delete(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodDelete,
-		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	body, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, "", cr.Text)
-	assert.True(t, cr.Deleted)
-}
-
-func TestAdmin_DeleteUser(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", Orig: "o test test #2", User: store.User{ID: "id2", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c3 := store.Comment{Text: "test test #3", Orig: "o test test #3", User: store.User{ID: "id2", Name: "name"}, ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	// write comments directly to store to keep user id
-	id1, err := srv.DataService.Create(c1)
-	assert.NoError(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.NoError(t, err)
-	_, err = srv.DataService.Create(c3)
-	assert.NoError(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	// all 3 comments here, but for id2 they deleted
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
-
-	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
-
-	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
-
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
-}
-
-func TestAdmin_Pin(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	pin := func(val int) int {
-		client := http.Client{}
-		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
-		resp, err := client.Do(req)
-		assert.Nil(t, err)
-		return resp.StatusCode
-	}
-
-	code := pin(1)
-	assert.Equal(t, 200, code)
-
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.True(t, cr.Pin)
-
-	code = pin(-1)
-	assert.Equal(t, 200, code)
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr = store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.False(t, cr.Pin)
-}
-
-func TestAdmin_Block(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	block := func(val int) (code int, body []byte) {
-		client := http.Client{}
-		req, e := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val), nil)
-		assert.Nil(t, e)
-		req.SetBasicAuth("dev", "password")
-		resp, e := client.Do(req)
-		require.Nil(t, e)
-		body, e = ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
-		resp.Body.Close()
-		return resp.StatusCode, body
-	}
-
-	code, body := block(1)
-	require.Equal(t, 200, code)
-	j := JSON{}
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, "user1", j["user_id"])
-	assert.Equal(t, true, j["block"])
-	assert.Equal(t, "radio-t", j["site_id"])
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
-	assert.True(t, comments.Comments[0].Deleted)
-
-	code, body = block(-1)
-	require.Equal(t, 200, code)
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, false, j["block"])
-}
-
-func TestAdmin_BlockedList(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	client := http.Client{}
-
-	// block user1
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	// block user2
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user2", 1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	res, code := getWithAuth(t, ts.URL+"/api/v1/admin/blocked?site=radio-t")
-	require.Equal(t, 200, code, res)
-	users := []store.BlockedUser{}
-	err = json.Unmarshal([]byte(res), &users)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(users), "two users blocked")
-	assert.Equal(t, "user1", users[0].ID)
-	assert.Equal(t, "user2", users[1].ID)
-}
-
-func TestAdmin_ReadOnly(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.False(t, info.ReadOnly)
-
-	client := http.Client{}
-
-	// set post to read-only
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-	// reset post's read-only
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.False(t, info.ReadOnly)
-}
-
-func TestAdmin_ReadOnlyWithAge(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"},
-		Timestamp: time.Date(2001, 1, 1, 1, 1, 1, 0, time.Local)}
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 10)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly, "ro by age")
-
-	client := http.Client{}
-
-	// set post to read-only
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-	// reset post's read-only
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-}
-func TestAdmin_Verify(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	verified := srv.DataService.IsVerified("radio-t", "user1")
-	assert.False(t, verified)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
-	assert.True(t, verified)
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
-	assert.True(t, comments.Comments[0].User.Verified)
-
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
-	assert.False(t, verified)
-
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments = commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
-	assert.False(t, comments.Comments[0].User.Verified)
-
-}
-
-func TestAdmin_ExportStream(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, 2, strings.Count(body, "\n"))
-	assert.Equal(t, 2, strings.Count(body, "\"text\""))
-	t.Logf("%s", body)
-}
-
-func TestAdmin_ExportFile(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
-	t.Logf("%s", string(ungzBody))
-}
-
-func TestAdmin_DeleteMeRequest(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	comments, err := srv.DataService.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "a comment for user1")
-
-	claims := auth.CustomClaims{
-		SiteID:      "radio-t",
-		SessionOnly: true,
-		StandardClaims: jwt.StandardClaims{
-			Id:        "1234567",
-			Issuer:    "remark42",
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-		},
-		User: &store.User{
-			ID: "user1",
-		},
-	}
-
-	token, err := srv.Authenticator.JWTService.Token(&claims)
-	assert.Nil(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	_, err = srv.DataService.User("radio-t", "user1", 0, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-}
-
-func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	// try with bad token
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, "bad token"), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
-
-	// try with bad auth
-	claims := auth.CustomClaims{
-		SiteID:      "radio-t",
-		SessionOnly: true,
-		StandardClaims: jwt.StandardClaims{
-			Id:        "1234567",
-			Issuer:    "remark42",
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-		},
-		User: &store.User{
-			ID: "user1",
-		},
-	}
-
-	token, err := srv.Authenticator.JWTService.Token(&claims)
-	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "bad-password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-
-	// try bad user
-	badClaims := claims
-	badClaims.User.ID = "no-such-id"
-	token, err = srv.Authenticator.JWTService.Token(&badClaims)
-	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode, resp.Status)
-
-}

app/rest/api/middleware.go

@@ -1,169 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"regexp"
-	"runtime/debug"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi/middleware"
-
-	"github.com/umputun/remark/app/rest"
-)
-
-// JSON is a map alias, just for convenience
-type JSON map[string]interface{}
-
-// AppInfo adds custom app-info to the response header
-func AppInfo(app string, version string) func(http.Handler) http.Handler {
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Org", "Umputun")
-			w.Header().Set("App-Name", app)
-			w.Header().Set("App-Version", version)
-			if mhost := os.Getenv("MHOST"); mhost != "" {
-				w.Header().Set("Host", mhost)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// Ping middleware response with pong to /ping. Stops chain if ping request detected
-func Ping(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/ping") {
-			w.Header().Set("Content-Type", "text/plain")
-			w.WriteHeader(http.StatusOK)
-			if _, err := w.Write([]byte("pong")); err != nil {
-				log.Printf("[WARN] can't send pong, %s", err)
-			}
-			return
-		}
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// Recoverer is a middleware that recovers from panics, logs the panic and returns a HTTP 500 status if possible.
-func Recoverer(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		defer func() {
-			if rvr := recover(); rvr != nil {
-				log.Printf("[WARN] request panic, %v", rvr)
-				debug.PrintStack()
-				http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			}
-		}()
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// LoggerFlag type
-type LoggerFlag int
-
-// logger flags enum
-const (
-	LogAll LoggerFlag = iota
-	LogUser
-	LogBody
-	LogNone
-)
-const maxBody = 1024
-
-var reMultWhtsp = regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-// Logger middleware prints http log. Customized by set of LoggerFlag
-func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if inLogFlags(LogNone, flags) { // skip logging
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			ww := middleware.NewWrapResponseWriter(w, 1)
-			body, user := getBodyAndUser(r, flags)
-			t1 := time.Now()
-			defer func() {
-				t2 := time.Now()
-
-				q := r.URL.String()
-				if qun, err := url.QueryUnescape(q); err == nil {
-					q = qun
-				}
-
-				remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-				if strings.HasPrefix(r.RemoteAddr, "[") {
-					remoteIP = strings.Split(r.RemoteAddr, "]:")[0] + "]"
-				}
-				if ipFn != nil {
-					remoteIP = ipFn(remoteIP)
-				}
-
-				log.Printf("[INFO] REST %s - %s - %s - %d (%d) - %v %s %s",
-					r.Method, q, remoteIP, ww.Status(), ww.BytesWritten(), t2.Sub(t1), user, body)
-			}()
-
-			h.ServeHTTP(ww, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-
-	return f
-}
-
-func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user string) {
-	ctx := r.Context()
-	if ctx == nil {
-		return "", ""
-	}
-
-	if inLogFlags(LogBody, flags) {
-		if content, err := ioutil.ReadAll(r.Body); err == nil {
-			body = string(content)
-			r.Body = ioutil.NopCloser(bytes.NewReader(content))
-
-			if len(body) > 0 {
-				body = strings.Replace(body, "\n", " ", -1)
-				body = reMultWhtsp.ReplaceAllString(body, " ")
-			}
-
-			if len(body) > maxBody {
-				body = body[:maxBody] + "..."
-			}
-		}
-	}
-
-	if inLogFlags(LogUser, flags) {
-		u, err := rest.GetUserInfo(r)
-		if err == nil && u.Name != "" {
-			user = fmt.Sprintf(" - %s %q", u.ID, u.Name)
-		}
-	}
-
-	return body, user
-}
-
-func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool {
-	for _, flg := range flags {
-		if (flg == LogAll && f != LogNone) || flg == f {
-			return true
-		}
-	}
-	return false
-}

app/rest/api/middleware_test.go

@@ -1,63 +0,0 @@
-package api
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestMiddleware_AppInfo(t *testing.T) {
-	router := chi.NewRouter()
-	router.With(AppInfo("remark42", "12345")).Get("/blah", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
-		w.Write([]byte("blah blah"))
-	})
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/blah")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "blah blah", string(b))
-	assert.Equal(t, "remark42", resp.Header.Get("App-Name"))
-	assert.Equal(t, "12345", resp.Header.Get("App-Version"))
-	assert.Equal(t, "Umputun", resp.Header.Get("Org"))
-}
-
-func TestMiddleware_GetBodyAndUser(t *testing.T) {
-	req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body"))
-	require.Nil(t, err)
-
-	body, user := getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, "body", body)
-	assert.Equal(t, "", user, "no user")
-
-	req = rest.SetUserInfo(req, store.User{ID: "id1", Name: "user1"})
-	_, user = getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-
-	body, user = getBodyAndUser(req, nil)
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogNone})
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogUser})
-	assert.Equal(t, "", body)
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-}

app/rest/api/migrator.go

@@ -1,133 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-)
-
-// Migrator rest runs on unexposed port and available for local requests only
-type Migrator struct {
-	Version        string
-	Cache          cache.LoadingCache
-	NativeImporter migrator.Importer
-	DisqusImporter migrator.Importer
-	NativeExported migrator.Exporter
-	SecretKey      string
-
-	httpServer *http.Server
-	lock       sync.Mutex
-}
-
-// Run the listener and request's router, activate rest server
-// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
-func (m *Migrator) Run(port int) {
-	log.Printf("[INFO] activate import server on port %d", port)
-	router := m.routes()
-
-	m.lock.Lock()
-	m.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
-	m.lock.Unlock()
-
-	err := m.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown import http server
-func (m *Migrator) Shutdown() {
-	log.Print("[WARN] shutdown import server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-
-	m.lock.Lock()
-	if err := m.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] importer shutdown error, %s", err)
-	}
-	m.lock.Unlock()
-
-	log.Print("[DEBUG] shutdown import server completed")
-}
-
-func (m *Migrator) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(15*time.Minute))
-	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-	router.Use(AppInfo("remark42-migrator", m.Version), Ping, Logger(nil, LogAll))
-	router.Post("/api/v1/admin/import", m.importCtrl)
-	router.Get("/api/v1/admin/export", m.exportCtrl)
-	return router
-}
-
-// POST /import?secret=key&site=site-id&provider=disqus|remark
-// imports comments from post body.
-func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-	importer := m.NativeImporter
-	if r.URL.Query().Get("provider") == "disqus" {
-		importer = m.DisqusImporter
-	}
-
-	size, err := importer.Import(r.Body, siteID)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
-		return
-	}
-	m.Cache.Flush(siteID)
-
-	render.Status(r, http.StatusCreated)
-	render.JSON(w, r, JSON{"status": "ok", "size": size})
-}
-
-// GET /export?site=site-id&secret=12345
-// exports all comments for siteID as gz file
-func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-
-	exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-	w.WriteHeader(http.StatusOK)
-	gzWriter := gzip.NewWriter(w)
-	defer func() {
-		if e := gzWriter.Close(); e != nil {
-			log.Printf("[WARN] can't close gzip writer, %s", e)
-		}
-	}()
-
-	if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
-}

app/rest/api/migrator_test.go

@@ -1,126 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-func TestMigrator_Import(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b))
-}
-
-func TestMigrator_ImportRejected(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-}
-
-func TestMigrator_Export(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	require.Nil(t, err)
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=123456", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
-	t.Logf("%s", string(ungzBody))
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=bad", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 403, resp.StatusCode)
-}
-
-func TestMigrator_Shutdown(t *testing.T) {
-	srv := Migrator{}
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b}
-	svc = &Migrator{
-		DisqusImporter: &migrator.Disqus{DataStore: dataStore},
-		NativeImporter: &migrator.Remark{DataStore: dataStore},
-		NativeExported: &migrator.Remark{DataStore: dataStore},
-		Cache:          &mockCache{},
-		SecretKey:      "123456",
-	}
-
-	routes := svc.routes()
-	ts = httptest.NewServer(routes)
-	return svc, ts
-}
-
-func cleanupImportSrv(srv *Migrator, ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-}

app/rest/api/rest.go

@@ -1,249 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-	"github.com/pkg/errors"
-	"gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Rest is a rest access server
-type Rest struct {
-	Version         string
-	DataService     *service.DataStore
-	Authenticator   auth.Authenticator
-	Exporter        migrator.Exporter
-	Cache           cache.LoadingCache
-	AvatarProxy     *proxy.Avatar
-	ImageProxy      *proxy.Image
-	WebRoot         string
-	RemarkURL       string
-	ReadOnlyAge     int
-	ScoreThresholds struct {
-		Low      int
-		Critical int
-	}
-
-	httpServer *http.Server
-	lock       sync.Mutex
-
-	adminService admin
-}
-
-const hardBodyLimit = 1024 * 64 // limit size of body
-
-var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
-	blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
-	blackfriday.BackslashLineBreak | blackfriday.Autolink
-
-type commentsWithInfo struct {
-	Comments []store.Comment `json:"comments"`
-	Info     store.PostInfo  `json:"info,omitempty"`
-}
-
-// Run the lister and request's router, activate rest server
-func (s *Rest) Run(port int) {
-	log.Printf("[INFO] activate rest server on port %d", port)
-
-	if len(s.Authenticator.Admins) > 0 {
-		log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
-	}
-
-	router := s.routes()
-
-	s.lock.Lock()
-	s.httpServer = &http.Server{
-		Addr:              fmt.Sprintf(":%d", port),
-		Handler:           router,
-		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      5 * time.Second,
-		IdleTimeout:       30 * time.Second,
-	}
-	s.lock.Unlock()
-
-	err := s.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown rest http server
-func (s *Rest) Shutdown() {
-	log.Print("[WARN] shutdown rest server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	s.lock.Lock()
-	if err := s.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] rest shutdown error, %s", err)
-	}
-	log.Print("[DEBUG] shutdown rest server completed")
-	s.lock.Unlock()
-}
-
-func (s *Rest) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
-	router.Use(AppInfo("remark42", s.Version), Ping)
-
-	s.adminService = admin{
-		dataService:   s.DataService,
-		exporter:      s.Exporter,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
-
-	ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization
-
-	// auth routes for all providers
-	router.Route("/auth", func(r chi.Router) {
-		r.Use(Logger(ipFn, LogAll), tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
-		for _, provider := range s.Authenticator.Providers {
-			r.Mount("/"+provider.Name, provider.Routes()) // mount auth providers as /auth/{name}
-		}
-		if len(s.Authenticator.Providers) > 0 {
-			// shortcut, can be any of providers, all logouts do the same - removes cookie
-			r.Get("/logout", s.Authenticator.Providers[0].LogoutHandler)
-		}
-	})
-
-	avatarMiddlewares := []func(http.Handler) http.Handler{
-		Logger(ipFn, LogNone),
-		tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)),
-	}
-	router.Mount(s.AvatarProxy.Routes(avatarMiddlewares...)) // mount avatars to /api/v1/avatar/{file.img}
-
-	// api routes
-	router.Route("/api/v1", func(rapi chi.Router) {
-		rapi.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-
-		// open routes
-		rapi.Group(func(ropen chi.Router) {
-			ropen.Use(s.Authenticator.Auth(false))
-			ropen.Use(Logger(ipFn, LogAll))
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
-			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
-			ropen.Get("/info", s.infoCtrl)
-
-			ropen.Mount("/rss", s.rssRoutes())
-			ropen.Mount("/img", s.ImageProxy.Routes())
-		})
-
-		// protected routes, require auth
-		rapi.Group(func(rauth chi.Router) {
-			rauth.Use(s.Authenticator.Auth(true))
-			rauth.Use(Logger(ipFn, LogAll))
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
-
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll)))
-		})
-	})
-
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
-	return router
-}
-
-// serves static files from /web
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
-	log.Printf("[INFO] run file server for %s, path %s", root, path)
-	origPath := path
-	fs := http.StripPrefix(path, http.FileServer(root))
-	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
-		path += "/"
-	}
-	path += "*"
-
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// don't show dirs, just serve files
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-				http.NotFound(w, r)
-				return
-			}
-			fs.ServeHTTP(w, r)
-		}))
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) {
-	data, err := encodeJSONWithHTML(v)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't render json response")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-func encodeJSONWithHTML(v interface{}) ([]byte, error) {
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false)
-	if err := enc.Encode(v); err != nil {
-		return nil, errors.Wrap(err, "json encoding failed")
-	}
-	return buf.Bytes(), nil
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONFromBytes(w http.ResponseWriter, r *http.Request, data []byte) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	if status, ok := r.Context().Value(render.StatusCtxKey).(int); ok {
-		w.WriteHeader(status)
-	}
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
-	for _, c := range comments {
-		if fn(c) {
-			filtered = append(filtered, c)
-		}
-	}
-	return filtered
-}

app/rest/api/rest_private.go

@@ -1,262 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"time"
-
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	multierror "github.com/hashicorp/go-multierror"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	comment := store.Comment{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	log.Printf("[DEBUG] create comment %+v", comment)
-
-	comment.PrepareUntrusted() // clean all fields user not supposed to set
-	comment.User = user
-	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
-
-	comment.Orig = comment.Text // original comment text, prior to md render
-	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
-		return
-	}
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
-	comment.Text = s.ImageProxy.Convert(comment.Text)
-	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
-		return
-	}
-
-	if s.ReadOnlyAge > 0 {
-		if info, e := s.DataService.Info(comment.Locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
-			return
-		}
-	}
-
-	id, err := s.DataService.Create(comment)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
-		return
-	}
-
-	// DataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
-		return
-	}
-	s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)
-
-	render.Status(r, http.StatusCreated)
-	render.JSON(w, r, &finalComment)
-}
-
-// PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	edit := struct {
-		Text    string
-		Summary string
-	}{}
-
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	id := chi.URLParam(r, "id")
-
-	log.Printf("[DEBUG] update comment %s", id)
-
-	var currComment store.Comment
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
-		return
-	}
-
-	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
-		return
-	}
-
-	text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
-	text = s.ImageProxy.Convert(text)
-	editReq := service.EditRequest{
-		Text:    text,
-		Orig:    edit.Text,
-		Summary: edit.Summary,
-	}
-
-	res, err := s.DataService.EditComment(locator, id, editReq)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
-		return
-	}
-
-	s.Cache.Flush(locator.URL, "last", user.ID)
-	render.JSON(w, r, res)
-}
-
-// GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-
-	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
-	}
-
-	render.JSON(w, r, user)
-}
-
-// PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	id := chi.URLParam(r, "id")
-	log.Printf("[DEBUG] vote for comment %s", id)
-
-	vote := r.URL.Query().Get("vote") == "1"
-
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
-		return
-	}
-	s.Cache.Flush(locator.URL)
-	render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
-}
-
-// GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	userB, err := json.Marshal(&user)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
-		return
-	}
-
-	exportFile := fmt.Sprintf("%s-%s-%s.json.gz", siteID, user.ID, time.Now().Format("20060102"))
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-	gzWriter := gzip.NewWriter(w)
-	defer func() {
-		if e := gzWriter.Close(); e != nil {
-			log.Printf("[WARN] can't close gzip writer, %s", e)
-		}
-	}()
-
-	write := func(val []byte) error {
-		_, e := gzWriter.Write(val)
-		return e
-	}
-
-	var merr error
-
-	merr = multierror.Append(merr, write([]byte(`{"info": `)))     // send user prefix
-	merr = multierror.Append(merr, write(userB))                   // send user info
-	merr = multierror.Append(merr, write([]byte(`, "comments":`))) // send comments prefix
-
-	// get comments in 100 in each paginated request
-	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
-			return
-		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
-			return
-		}
-
-		merr = multierror.Append(merr, write(b))
-		if len(comments) != 100 {
-			break
-		}
-	}
-
-	merr = multierror.Append(merr, write([]byte(`}`)))
-	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
-		return
-	}
-
-}
-
-// POST /deleteme?site_id=site - requesting delete of all user info
-// makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	siteID := r.URL.Query().Get("site")
-
-	claims := auth.CustomClaims{
-		SiteID: siteID,
-		StandardClaims: jwt.StandardClaims{
-			Issuer:    "remark42",
-			ExpiresAt: time.Now().AddDate(0, 3, 0).Unix(),
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-		},
-		User: &user,
-	}
-
-	tokenStr, err := s.Authenticator.JWTService.Token(&claims)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
-		return
-	}
-	link := fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", s.RemarkURL, tokenStr)
-	render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
-}

app/rest/api/rest_private_test.go

@@ -1,385 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestRest_Create(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	loc := c["locator"].(map[string]interface{})
-	assert.Equal(t, "radio-t", loc["site"])
-	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
-	assert.True(t, len(c["id"].(string)) > 8)
-}
-
-func TestRest_CreateOldPost(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// make old, but not too old comment
-	old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(old)
-	assert.Nil(t, err)
-
-	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments))
-
-	// try to add new comment to the same old post
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	assert.Nil(t, srv.DataService.DeleteAll("radio-t"))
-	// make too old comment
-	old = store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(old)
-	assert.Nil(t, err)
-
-	resp, err = post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-}
-
-func TestRest_CreateTooBig(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
-
-	resp, err := post(t, ts.URL+"/api/v1/comment", longComment)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	assert.Equal(t, "comment text exceeded max allowed size 4000 (4001)", c["error"])
-	assert.Equal(t, "invalid comment", c["details"])
-
-	veryLongComment := fmt.Sprintf(`{"text": "%70000s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
-	resp, err = post(t, ts.URL+"/api/v1/comment", veryLongComment)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c = JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	assert.Equal(t, "http: request body too large", c["error"])
-	assert.Equal(t, "can't bind comment", c["details"])
-}
-
-func TestRest_CreateRejected(t *testing.T) {
-
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-	body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`
-
-	// try to create without auth
-	resp, err := http.Post(ts.URL+"/api/v1/comment", "", strings.NewReader(body))
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-}
-func TestRest_CreateAndGet(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// create comment
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "**test** *123* http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	require.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-
-	id := c["id"].(string)
-
-	// get created comment by id
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
-	assert.Equal(t, 200, code)
-	comment := store.Comment{}
-	err = json.Unmarshal([]byte(res), &comment)
-	assert.Nil(t, err)
-	assert.Equal(t, `<p><strong>test</strong> <em>123</em> <a href="http://radio-t.com" rel="nofollow">http://radio-t.com</a></p>`+"\n", comment.Text)
-	assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
-		comment.User)
-	t.Logf("%+v", comment)
-}
-
-func TestRest_Update(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id := addComment(t, c1, ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
-		strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err := client.Do(req)
-	assert.Nil(t, err)
-	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, b.StatusCode, string(body))
-
-	// comments returned by update
-	c2 := store.Comment{}
-	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
-	assert.Equal(t, id, c2.ID)
-	assert.Equal(t, "<p>updated text</p>\n", c2.Text)
-	assert.Equal(t, "updated text", c2.Orig)
-	assert.Equal(t, "my edit", c2.Edit.Summary)
-	assert.True(t, time.Since(c2.Edit.Timestamp) < 1*time.Second)
-
-	// read updated comment
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
-	assert.Equal(t, 200, code)
-	c3 := store.Comment{}
-	err = json.Unmarshal([]byte(res), &c3)
-	assert.Nil(t, err)
-	assert.Equal(t, c2, c3, "same as response from update")
-}
-
-func TestRest_UpdateNotOwner(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
-	id1, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err := client.Do(req)
-	assert.Nil(t, err)
-	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
-
-	client = http.Client{}
-	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
-}
-
-func TestRest_Vote(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	vote := func(val int) int {
-		client := http.Client{}
-		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
-		resp, err := client.Do(req)
-		assert.Nil(t, err)
-		return resp.StatusCode
-	}
-
-	assert.Equal(t, 200, vote(1), "first vote allowed")
-	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"dev": true}, cr.Votes)
-
-	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr = store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
-}
-
-func TestRest_UserAllData(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// write 3 comments
-	user := store.User{ID: "dev", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
-	assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
-	t.Logf("%s", string(ungzBody))
-
-	parsed := struct {
-		Info     store.User      `json:"info"`
-		Comments []store.Comment `json:"comments"`
-	}{}
-
-	err = json.Unmarshal(ungzBody, &parsed)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true}, parsed.Info)
-	assert.Equal(t, 3, len(parsed.Comments))
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 401, resp.StatusCode)
-}
-
-func TestRest_UserAllDataManyComments(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	user := store.User{ID: "dev", Name: "user name 1"}
-	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-
-	for i := 0; i < 478; i++ {
-		c.ID = fmt.Sprintf("id-%03d", i)
-		c.Timestamp = c.Timestamp.Add(time.Second)
-		_, err := srv.DataService.Create(c)
-		require.Nil(t, err)
-	}
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
-}
-
-func TestRest_DeleteMe(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	m := map[string]string{}
-	err = json.Unmarshal(body, &m)
-	assert.Nil(t, err)
-	assert.Equal(t, "radio-t", m["site"])
-	assert.Equal(t, "dev", m["user_id"])
-
-	token := m["token"]
-	claims, err := srv.Authenticator.JWTService.Parse(token)
-	assert.Nil(t, err)
-	assert.Equal(t, "dev", claims.User.ID)
-	assert.Equal(t, "https://demo.remark42.com/api/v1/admin/deleteme?token="+token, m["link"])
-
-	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
-	assert.Nil(t, err)
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-}

app/rest/api/rest_public.go

@@ -1,311 +0,0 @@
-package api
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-)
-
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
-// find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	sort := r.URL.Query().Get("sort")
-	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
-		sort = "+" + sort[1:]
-	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, sort)
-		if e != nil {
-			return nil, e
-		}
-		maskedComments := s.adminService.alterComments(comments, r)
-		var b []byte
-		switch r.URL.Query().Get("format") {
-		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
-			if s.DataService.IsReadOnly(locator) {
-				tree.Info.ReadOnly = true
-			}
-			b, e = encodeJSONWithHTML(tree)
-		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
-				withInfo.Info = info
-			}
-			b, e = encodeJSONWithHTML(withInfo)
-		}
-		return b, e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
-	comment := store.Comment{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	comment.User = user
-	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
-		return
-	}
-
-	//comment.Text = string(blackfriday.Run([]byte(comment.Text),
-	//	blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
-	comment.Text = s.ImageProxy.Convert(comment.Text)
-	comment.Sanitize()
-	render.HTML(w, r, comment.Text)
-}
-
-// GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(info)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
-		return
-	}
-
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[DEBUG] get last comments for %s", siteID)
-
-	limit, err := strconv.Atoi(chi.URLParam(r, "limit"))
-	if err != nil {
-		limit = 0
-	}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-
-		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
-		filterDeleted := []store.Comment{}
-		for _, c := range comments {
-			if c.Deleted {
-				continue
-			}
-			filterDeleted = append(filterDeleted, c)
-		}
-
-		return encodeJSONWithHTML(filterDeleted)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
-
-	id := chi.URLParam(r, "id")
-	siteID := r.URL.Query().Get("site")
-	url := r.URL.Query().Get("url")
-
-	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
-
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
-		return
-	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
-	render.Status(r, http.StatusOK)
-	renderJSONWithHTML(w, r, comment)
-}
-
-// GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-
-	userID := r.URL.Query().Get("user")
-	siteID := r.URL.Query().Get("site")
-
-	limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
-	if err != nil {
-		limit = 0
-	}
-
-	resp := struct {
-		Comments []store.Comment `json:"comments,omitempty"`
-		Count    int             `json:"count,omitempty"`
-	}{}
-
-	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
-		if e != nil {
-			return nil, e
-		}
-		resp.Comments, resp.Count = comments, count
-		return encodeJSONWithHTML(resp)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.Authenticator.Admins,
-		AdminEmail:     s.Authenticator.AdminEmail,
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		ReadOnlyAge:    s.ReadOnlyAge,
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers {
-		cnf.Auth = append(cnf.Auth, ap.Name)
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
-// GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
-		return
-	}
-	render.JSON(w, r, JSON{"count": count, "locator": locator})
-}
-
-// POST /count?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	posts := []string{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
-		return
-	}
-
-	// key could be long for multiple posts, make it sha1
-	key := cache.URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
-	if _, err := hasher.Write([]byte(key)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
-
-	data, err := s.Cache.Get(cache.Key(sha, siteID), func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(counts)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
-
-	siteID := r.URL.Query().Get("site")
-	limit, skip := 0, 0
-
-	if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil {
-		limit = v
-	}
-	if v, err := strconv.Atoi(r.URL.Query().Get("skip")); err == nil {
-		skip = v
-	}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(posts)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}

app/rest/api/rest_public_test.go

@@ -1,449 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestRest_Ping(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	res, code := get(t, ts.URL+"/api/v1/ping")
-	assert.Equal(t, "pong", res)
-	assert.Equal(t, 200, code)
-}
-
-func TestRest_Preview(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "<p>test 123</p>\n", string(b))
-}
-
-func TestRest_PreviewWithMD(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	text := `
-# h1
-
-BKT
-func TestRest_Preview(t *testing.T) {
-srv, ts := prep(t)
-  require.NotNil(t, srv)
-}
-BKT
-`
-	text = strings.Replace(text, "BKT", "```", -1)
-	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
-	j = strings.Replace(j, "\n", "\\n", -1)
-	t.Log(j)
-
-	resp, err := post(t, ts.URL+"/api/v1/preview", j)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
-}
-
-func TestRest_Find(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id1 := addComment(t, c1, ts)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id2 := addComment(t, c2, ts)
-
-	assert.NotEqual(t, id1, id2)
-
-	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, id1, comments.Comments[0].ID)
-	assert.Equal(t, id2, comments.Comments[1].ID)
-	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
-	assert.Equal(t, 2, comments.Info.Count)
-	assert.Equal(t, false, comments.Info.ReadOnly)
-	assert.True(t, comments.Info.FirstTS.Before(comments.Info.LastTS))
-
-	// get sorted by -time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=-time")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, id1, comments.Comments[1].ID)
-	assert.Equal(t, id2, comments.Comments[0].ID)
-
-	// get in tree mode
-	tree := rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(tree.Nodes))
-	assert.Equal(t, 1, len(tree.Nodes[0].Replies))
-	assert.Equal(t, 2, tree.Info.Count)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is fresh")
-}
-
-func TestRest_FindAge(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-
-	tree := rest.Tree{}
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is fresh")
-
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
-	assert.True(t, tree.Info.ReadOnly, "post is old")
-}
-
-func TestRest_FindReadOnly(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(c1)
-
-	require.Nil(t, err)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -2),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-
-	// set post to read-only
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	tree := rest.Tree{}
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.True(t, tree.Info.ReadOnly, "post is ro")
-
-	tree = rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is writable")
-}
-
-func TestRest_Last(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	// add 3 comments
-	addComment(t, c1, ts)
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
-	assert.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
-	assert.Equal(t, id1, comments[1].ID)
-	assert.Equal(t, id2, comments[0].ID)
-
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-
-	res, code = get(t, ts.URL+"/api/v1/last/X?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-
-	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
-	assert.Nil(t, err)
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
-}
-
-func TestRest_FindUserComments(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	// add 3 comments
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	// add one deleted
-	id := addComment(t, c2, ts)
-	err := srv.DataService.Delete(c2.Locator, id, store.SoftDelete)
-	assert.NoError(t, err)
-
-	_, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=blah")
-	assert.Equal(t, 400, code, "noting for user blah")
-
-	res, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=dev")
-	assert.Equal(t, 200, code)
-
-	resp := struct {
-		Comments []store.Comment
-		Count    int
-	}{}
-
-	err = json.Unmarshal([]byte(res), &resp)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(resp.Comments), "should have 3 comments")
-	assert.Equal(t, 4, resp.Count, "should have 3 count")
-}
-
-func TestRest_UserInfo(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := getWithAuth(t, ts.URL+"/api/v1/user?site=radio-t")
-	assert.Equal(t, 200, code)
-	user := store.User{}
-	err := json.Unmarshal([]byte(body), &user)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: ""}, user)
-}
-
-func TestRest_Count(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	j := JSON{}
-	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 3.0, j["count"])
-
-	body, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah2")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 2.0, j["count"])
-}
-
-func TestRest_Counts(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	j := []store.PostInfo{}
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
-		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
-}
-
-func TestRest_List(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t")
-	assert.Equal(t, 200, code)
-	pi := []store.PostInfo{}
-	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
-	assert.Equal(t, 2, pi[0].Count)
-	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
-	assert.Equal(t, 3, pi[1].Count)
-}
-
-func TestRest_Config(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
-	assert.Equal(t, 200, code)
-	j := JSON{}
-	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 300., j["edit_duration"])
-	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
-	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
-	assert.Equal(t, 4000., j["max_comment_size"])
-	assert.Equal(t, -5., j["low_score"])
-	assert.Equal(t, -10., j["critical_score"])
-	assert.Equal(t, 10., j["readonly_age"])
-	t.Logf("%+v", j)
-}
-
-func TestRest_Info(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
-
-	user := store.User{ID: "user1", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
-
-	body, code := get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-
-	info := store.PostInfo{}
-	err = json.Unmarshal([]byte(body), &info)
-	assert.Nil(t, err)
-	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
-		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-	assert.Equal(t, exp, info)
-
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah-no")
-	assert.Equal(t, 400, code)
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
-	assert.Equal(t, 400, code)
-}
-
-func TestRest_Robots(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/robots.txt")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
-		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
-}

app/rest/api/rest_test.go

@@ -1,146 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-var testHTML = "/tmp/test-remark.html"
-
-func TestRest_FileServer(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/web/test-remark.html")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, "some html", body)
-}
-
-func TestRest_Shutdown(t *testing.T) {
-	srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300),
-		RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, Secret: "123456"}
-	srv = &Rest{
-		DataService: dataStore,
-		Authenticator: auth.Authenticator{
-			DevPasswd:  "password",
-			Providers:  nil,
-			Admins:     []string{"a1", "a2"},
-			AdminEmail: "admin@remark-42.com",
-			JWTService: auth.NewJWT("12345", false, time.Minute),
-		},
-		Exporter:    &migrator.Remark{DataStore: dataStore},
-		Cache:       &mockCache{},
-		WebRoot:     "/tmp",
-		RemarkURL:   "https://demo.remark42.com",
-		AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300), RoutePath: "/api/v1/avatar"},
-		ImageProxy:  &proxy.Image{},
-		ReadOnlyAge: 10,
-	}
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
-
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-	ts = httptest.NewServer(srv.routes())
-	return srv, ts
-}
-
-func get(t *testing.T, url string) (string, int) {
-	r, err := http.Get(url)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func getWithAuth(t *testing.T, url string) (string, int) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	r, err := client.Do(req)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func post(t *testing.T, url string, body string) (*http.Response, error) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	return client.Do(req)
-}
-
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
-
-	b, err := json.Marshal(c)
-	assert.Nil(t, err, "can't marshal comment %+v", c)
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	crResp := JSON{}
-	err = json.Unmarshal(b, &crResp)
-	assert.Nil(t, err)
-	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
-}
-
-func cleanup(ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-	os.Remove(testHTML)
-}
-
-type mockCache struct{}
-
-func (mc *mockCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
-	return fn()
-}
-
-func (mc *mockCache) Flush(scopes ...string) {}

app/rest/api/rss.go

@@ -1,129 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/gorilla/feeds"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-)
-
-const maxRssItems = 20
-
-// ui uses links like <post-url>#remark42__comment-<comment-id>
-const uiNav = "#remark42__comment-"
-
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	return router
-}
-
-// GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	log.Printf("[DEBUG] get rss for post %+v", locator)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments)
-		if e != nil {
-			return nil, e
-		}
-		return []byte(rss), e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-// GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[DEBUG] get rss for site %s", siteID)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
-		if e != nil {
-			return nil, e
-		}
-		return []byte(rss), e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
-
-	lastCommentTS := time.Unix(0, 0)
-	if len(comments) > 0 {
-		lastCommentTS = comments[0].Timestamp
-	}
-
-	feed := &feeds.Feed{
-		Title:       "Remark42 comments",
-		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
-		Created:     lastCommentTS,
-	}
-
-	feed.Items = []*feeds.Item{}
-	for i, c := range comments {
-		f := feeds.Item{
-			Title:       c.User.Name,
-			Link:        &feeds.Link{Href: c.Locator.URL + uiNav + c.ID},
-			Description: c.Text,
-			Created:     c.Timestamp,
-			Author:      &feeds.Author{Name: c.User.Name},
-		}
-		if c.ParentID != "" {
-			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
-			if err == nil {
-				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
-			} else {
-				log.Printf("[WARN] failed to get info about parent comment, %s", err)
-			}
-		}
-		feed.Items = append(feed.Items, &f)
-		if i > maxRssItems {
-			break
-		}
-	}
-	return feed.ToRss()
-}

app/rest/api/rss_test.go

@@ -1,181 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestServer_RssPost(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-        <channel>
-            <title>Remark42 comments</title>
-            <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
-            <pubDate>%s</pubDate>
-            <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-            </item>
-         </channel>
-	</rss>`, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssSite(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah11#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/site?site=bad-radio-t")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssWithReply(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	c2.ParentID = id1
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one &gt; developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-}
-
-func waitOnSecChange() {
-	for {
-		if time.Now().Nanosecond() < 100000000 {
-			break
-		}
-		time.Sleep(10 * time.Nanosecond)
-	}
-}
-
-// clean formatting, i.e. multiple spaces, \t, \n
-func cleanRssFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/auth/auth.go

@@ -1,132 +0,0 @@
-// Package auth provides oauth2 support as well as related middlewares.
-package auth
-
-import (
-	"encoding/base64"
-	"log"
-	"net/http"
-	"strings"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Authenticator is top level auth object providing middlewares
-type Authenticator struct {
-	JWTService *JWT
-	Providers  []Provider
-	Admins     []string
-	AdminEmail string
-	DevPasswd  string
-}
-
-var devUser = store.User{
-	ID:      "dev",
-	Name:    "developer one",
-	Picture: "/api/v1/avatar/remark.image",
-	Admin:   true,
-}
-
-// Auth middleware adds auth from session and populates user info
-func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if a.basicDevUser(w, r) { // fail-back to dev user if enabled
-				user := devUser
-				r = rest.SetUserInfo(r, user)
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			claims, err := a.JWTService.Get(r)
-			if err != nil && reqAuth { // in full auth lack of session causes Unauthorized
-				log.Printf("[DEBUG] failed auth, %s", err)
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if err != nil { // in anonymous mode just pass it to the next handler
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			if claims.User == nil && reqAuth {
-				log.Print("[DEBUG] failed auth, no user info presented in the claim")
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if claims.User != nil { // if uinfo in token populate it to context
-				user := *claims.User
-				user.Admin = isAdmin(user.ID, a.Admins) // dbl-check for admin to reset admin flag even if token has it
-				// refresh token if it close to expiration
-				if _, err := a.JWTService.Refresh(w, r); err != nil {
-					log.Printf("[DEBUG] can't refresh jwt, %s", err)
-				}
-				r = rest.SetUserInfo(r, user)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// AdminOnly allows access to admins
-func (a *Authenticator) AdminOnly(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		user, err := rest.GetUserInfo(r)
-		if err != nil {
-			http.Error(w, "Unauthorized", http.StatusUnauthorized)
-			return
-		}
-
-		if !user.Admin {
-			http.Error(w, "Access denied", http.StatusForbidden)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) bool {
-
-	if a.DevPasswd == "" {
-		return false
-	}
-
-	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
-	if len(s) != 2 {
-		return false
-	}
-
-	b, err := base64.StdEncoding.DecodeString(s[1])
-	if err != nil {
-		return false
-	}
-
-	pair := strings.SplitN(string(b), ":", 2)
-	if len(pair) != 2 {
-		return false
-	}
-
-	if pair[0] != "dev" || pair[1] != a.DevPasswd {
-		return false
-	}
-
-	return true
-}
-
-func isAdmin(userID string, admins []string) bool {
-	for _, admin := range admins {
-		if admin == userID {
-			return true
-		}
-	}
-	return false
-}

app/rest/auth/auth_test.go

@@ -1,171 +0,0 @@
-package auth
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAuthJWTCookie(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	expiration := int(time.Duration(365 * 24 * time.Hour).Seconds())
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "wrong id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "xsrf mismatch")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtExpired, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "token expired")
-}
-
-func TestAuthJWTHeader(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtValid)
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtExpired)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "invalid auth token")
-}
-func TestAuthRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "xyz")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAuthNotRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(false)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "ZZZZ123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAdminRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user, admin")
-
-	devUser.Admin = false
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 403, resp.StatusCode, "valid auth user, not admin")
-
-}
-func withBasicAuth(r *http.Request, username, password string) *http.Request {
-	auth := username + ":" + password
-	r.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
-	return r
-}

app/rest/auth/jwt.go

@@ -1,166 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"time"
-
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// JWT wraps jwt operations
-// supports both header and cookie jwt
-type JWT struct {
-	secret        string
-	secureCookies bool
-	exp           time.Duration
-}
-
-// CustomClaims stores user info for auth and state & from from login
-type CustomClaims struct {
-	jwt.StandardClaims
-	User *store.User `json:"user,omitempty"`
-
-	// state and from used for oauth handshake
-	State       string `json:"state,omitempty"`
-	From        string `json:"from,omitempty"`
-	SiteID      string `json:"site_id,omitempty"`
-	SessionOnly bool   `json:"sess_only,omitempty"`
-}
-
-const jwtCookieName = "JWT"
-const jwtHeaderKey = "X-JWT"
-const xsrfCookieName = "XSRF-TOKEN"
-const xsrfHeaderKey = "X-XSRF-TOKEN"
-
-// NewJWT makes JWT service
-func NewJWT(secret string, secureCookies bool, exp time.Duration) *JWT {
-	res := JWT{
-		secret:        secret,
-		secureCookies: secureCookies,
-		exp:           exp,
-	}
-	return &res
-}
-
-// Token makes jwt with claims
-func (j *JWT) Token(claims *CustomClaims) (string, error) {
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
-	tokenString, err := token.SignedString([]byte(j.secret))
-	if err != nil {
-		return "", errors.Wrap(err, "can't sign jwt token")
-	}
-	return tokenString, nil
-}
-
-// Parse token string and verify
-func (j *JWT) Parse(tokenString string) (*CustomClaims, error) {
-	token, err := jwt.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) {
-		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
-			return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"])
-		}
-		return []byte(j.secret), nil
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "can't parse jwt")
-	}
-
-	claims, ok := token.Claims.(*CustomClaims)
-	if !ok || !token.Valid {
-		return nil, errors.New("invalid jwt")
-	}
-	return claims, nil
-}
-
-// Set creates jwt cookie with xsrf cookie and put it to ResponseWriter
-// accepts claims and sets expiration if none defined. permanent flag means long-living cookie, false makes it session only.
-func (j *JWT) Set(w http.ResponseWriter, claims *CustomClaims, sessionOnly bool) error {
-	if claims.ExpiresAt == 0 {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-	}
-
-	tokenString, err := j.Token(claims)
-	if err != nil {
-		return errors.Wrap(err, "failed to make jwt token")
-	}
-
-	cookieExpiration := 0 // session cookie
-	if !sessionOnly {
-		cookieExpiration = 365 * 24 * 3600 // 1 year
-	}
-
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: tokenString, HttpOnly: true, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: claims.Id, HttpOnly: false, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-
-	return nil
-}
-
-// Get jwt from header or cookie
-// if cookie used, verify xsrf token to match
-func (j *JWT) Get(r *http.Request) (*CustomClaims, error) {
-
-	fromCookie := false
-	tokenString := ""
-
-	// try to get from X-JWT header
-	if tokenHeader := r.Header.Get(jwtHeaderKey); tokenHeader != "" {
-		tokenString = tokenHeader
-	}
-
-	// try to get from JWT cookie
-	if tokenString == "" {
-		fromCookie = true
-		jc, err := r.Cookie(jwtCookieName)
-		if err != nil {
-			return nil, errors.Wrap(err, "jwt cookie was not presented")
-		}
-		tokenString = jc.Value
-	}
-
-	claims, err := j.Parse(tokenString)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get jwt")
-	}
-
-	if fromCookie && claims.User != nil {
-		xsrf := r.Header.Get(xsrfHeaderKey)
-		if claims.Id != xsrf {
-			return nil, errors.New("xsrf mismatch")
-		}
-	}
-
-	return claims, nil
-}
-
-// Refresh gets jwt from request, checks if it will be expiring soon (1/2 of expiration) and create the new onw
-func (j *JWT) Refresh(w http.ResponseWriter, r *http.Request) (*CustomClaims, error) {
-	claims, err := j.Get(r)
-	if err != nil {
-		return nil, err
-	}
-	untilExp := claims.ExpiresAt - time.Now().Unix()
-	if untilExp <= int64(j.exp.Seconds()/2) {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-		e := j.Set(w, claims, claims.SessionOnly)
-		return claims, e
-	}
-	return claims, nil
-}
-
-// Reset token's cookies
-func (j *JWT) Reset(w http.ResponseWriter) {
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-}

app/rest/auth/jwt_test.go

@@ -1,287 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCI" +
-	"sImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZS" +
-	"I6IiIsImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ._loFgh3g45gr9TtGqvM3N584I_6EHEOJnYb6Py84stQ"
-
-var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6In" +
-	"JlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsIm" +
-	"FkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWV9.p6w0sM_NYaRuyhyA9jqfWlB5cx1vZPGhXGC5geSX7nA"
-
-var testJwtExpired = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MjY4ODc4MjIsImp0aSI6InJhbmRvbSBpZCIs" +
-	"ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiI" +
-	"sImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ.4_dCrY9ihyfZIedz-kZwBTxmxU1a52V7IqeJrOqTzE4"
-
-func TestJWT_Token(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	res, err := j.Token(claims)
-	assert.Nil(t, err)
-	assert.Equal(t, testJwtValid, res)
-}
-
-func TestJWT_Parse(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-	claims, err := j.Parse(testJwtValid)
-	assert.NoError(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1"}, claims.User)
-
-	_, err = j.Parse(testJwtExpired)
-	assert.NotNil(t, err, "expired token")
-
-	_, err = j.Parse("bad")
-	assert.NotNil(t, err, "bad token")
-}
-
-func TestJWT_Set(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	claims.SessionOnly = false
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValid, cookies[0].Value)
-	assert.Equal(t, 31536000, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-
-	claims.SessionOnly = true
-	rr = httptest.NewRecorder()
-	err = j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies = rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValidSess, cookies[0].Value)
-	assert.Equal(t, 0, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-}
-
-func TestJWT_GetFromHeader(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	req := httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtValid)
-	claims, err := j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtExpired)
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, "bad bad token")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token contains an invalid number of segments"), err.Error())
-
-}
-
-func TestJWT_SetAndGetWithCookies(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State:       "123456",
-		From:        "from",
-		SessionOnly: true,
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	claims, err = j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-	assert.Equal(t, true, claims.SessionOnly)
-	t.Log(resp.Cookies())
-}
-
-func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id wrong")
-	claims, err = j.Get(req)
-	assert.EqualError(t, err, "xsrf mismatch")
-}
-
-func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2018, 5, 21, 1, 35, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/expired" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/expired")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/expired", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-}
-
-func TestJWT_Refresh(t *testing.T) {
-	j := NewJWT("xyz 12345", false, 2*time.Second)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:     "random id",
-			Issuer: "remark42",
-		},
-	}
-	// set token
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, true)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	require.Equal(t, 2, len(cookies))
-
-	req, err := http.NewRequest("GET", "http://example.com/blah", nil)
-	require.Nil(t, err)
-	req.AddCookie(cookies[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-
-	claims2, err := j.Refresh(rr, req)
-	require.Nil(t, err)
-	assert.Equal(t, claims.ExpiresAt, claims2.ExpiresAt, "no refresh yet")
-
-	time.Sleep(1 * time.Second)
-	claims2, err = j.Refresh(rr, req)
-	assert.Nil(t, err)
-	assert.True(t, claims.ExpiresAt < claims2.ExpiresAt, "refreshed")
-	t.Log(claims.ExpiresAt, claims2.ExpiresAt)
-}

app/rest/auth/provider.go

@@ -1,224 +0,0 @@
-package auth
-
-import (
-	"context"
-	"crypto/rand"
-	"crypto/sha1"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-)
-
-// Provider represents oauth2 provider
-type Provider struct {
-	Params
-	Name        string
-	RedirectURL string
-	InfoURL     string
-	Endpoint    oauth2.Endpoint
-	Scopes      []string
-	MapUser     func(userData, []byte) store.User // map info from InfoURL to User
-	conf        oauth2.Config
-}
-
-// Params to make initialized and ready to use provider
-type Params struct {
-	RemarkURL    string
-	AvatarProxy  *proxy.Avatar
-	JwtService   *JWT
-	IsVerifiedFn func(siteID string, userID string) bool
-	SecretKey    string
-	Admins       []string
-	Cid          string
-	Csecret      string
-}
-
-type userData map[string]interface{}
-
-func (u userData) value(key string) string {
-	// json.Unmarshal converts json "null" value to go's "nil", in this case return empty string
-	if val, ok := u[key]; ok && val != nil {
-		return fmt.Sprintf("%v", val)
-	}
-	return ""
-}
-
-// newProvider makes auth for given provider
-func initProvider(p Params, provider Provider) Provider {
-	log.Printf("[INFO] init auth provider %s", provider.Name)
-	provider.Params = p
-	provider.conf = oauth2.Config{
-		ClientID:     provider.Cid,
-		ClientSecret: provider.Csecret,
-		RedirectURL:  provider.RedirectURL,
-		Scopes:       provider.Scopes,
-		Endpoint:     provider.Endpoint,
-	}
-
-	log.Printf("[DEBUG] created %s auth, id=%s, redir=%s, endpoint=%s",
-		provider.Name, provider.Cid, provider.Endpoint, provider.RedirectURL)
-	return provider
-}
-
-// Routes returns auth routes for given provider
-func (p Provider) Routes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/login", p.loginHandler)
-	router.Get("/callback", p.authHandler)
-	router.Get("/logout", p.LogoutHandler)
-	return router
-}
-
-// loginHandler - GET /login?from=redirect-back-url&site=siteID&session=1
-func (p Provider) loginHandler(w http.ResponseWriter, r *http.Request) {
-
-	log.Printf("[DEBUG] login with %s", p.Name)
-	// make state (random) and store in session
-	state := p.randToken()
-
-	claims := CustomClaims{
-		State:       state,
-		From:        r.URL.Query().Get("from"),
-		SiteID:      r.URL.Query().Get("site"),
-		SessionOnly: r.URL.Query().Get("session") != "" && r.URL.Query().Get("session") != "0",
-		StandardClaims: jwt.StandardClaims{
-			Id:        p.randToken(),
-			Issuer:    "remark42",
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-		},
-	}
-
-	if err := p.JwtService.Set(w, &claims, false); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set jwt")
-		return
-	}
-
-	// return login url
-	loginURL := p.conf.AuthCodeURL(state)
-	log.Printf("[DEBUG] login url %s", loginURL)
-
-	http.Redirect(w, r, loginURL, http.StatusFound)
-}
-
-// authHandler fills user info and redirects to "from" url. This is callback url redirected locally by browser
-// GET /callback
-func (p Provider) authHandler(w http.ResponseWriter, r *http.Request) {
-
-	oauthClaims, err := p.JwtService.Get(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to get jwt")
-		return
-	}
-
-	retrievedState := oauthClaims.State
-	if retrievedState == "" || retrievedState != r.URL.Query().Get("state") {
-		http.Error(w, fmt.Sprintf("unexpected state %v", retrievedState), http.StatusUnauthorized)
-		return
-	}
-
-	log.Printf("[DEBUG] auth with state %s", retrievedState)
-	tok, err := p.conf.Exchange(context.Background(), r.URL.Query().Get("code"))
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "exchange failed")
-		return
-	}
-
-	client := p.conf.Client(context.Background(), tok)
-	uinfo, err := client.Get(p.InfoURL)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, fmt.Sprintf("failed to get client info via %s", p.InfoURL))
-		return
-	}
-
-	defer func() {
-		if e := uinfo.Body.Close(); e != nil {
-			log.Printf("[WARN] failed to close response body, %s", e)
-		}
-	}()
-
-	data, err := ioutil.ReadAll(uinfo.Body)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to read user info")
-		return
-	}
-
-	jData := map[string]interface{}{}
-	if e := json.Unmarshal(data, &jData); e != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to unmarshal user info")
-		return
-	}
-	log.Printf("[DEBUG] got raw user info %+v", jData)
-
-	u := p.MapUser(jData, data)
-	u = p.alterUser(u, oauthClaims)
-
-	authClaims := &CustomClaims{
-		User: &u,
-		StandardClaims: jwt.StandardClaims{
-			Issuer: "remark42",
-			Id:     p.randToken(),
-		},
-		SessionOnly: oauthClaims.SessionOnly,
-	}
-
-	if err = p.JwtService.Set(w, authClaims, oauthClaims.SessionOnly); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to save user info")
-		return
-	}
-
-	log.Printf("[DEBUG] user info %+v", u)
-
-	// redirect to back url if presented in login query params
-	if oauthClaims.From != "" {
-		http.Redirect(w, r, oauthClaims.From, http.StatusTemporaryRedirect)
-		return
-	}
-	render.JSON(w, r, &u)
-}
-
-// alterUser sets fields not handled by provider's MapUser, things like avatar, admin, verified
-func (p Provider) alterUser(u store.User, oauthClaims *CustomClaims) store.User {
-	if p.AvatarProxy != nil {
-		if avatarURL, e := p.AvatarProxy.Put(u); e == nil {
-			u.Picture = avatarURL
-		} else {
-			log.Printf("[WARN] failed to proxy avatar, %s", e)
-		}
-	}
-	u.Admin = isAdmin(u.ID, p.Admins)
-	if p.IsVerifiedFn != nil {
-		u.Verified = p.IsVerifiedFn(oauthClaims.SiteID, u.ID)
-	}
-	return u
-}
-
-// LogoutHandler - GET /logout
-func (p Provider) LogoutHandler(w http.ResponseWriter, r *http.Request) {
-	p.JwtService.Reset(w)
-	log.Printf("[DEBUG] logout")
-}
-
-func (p Provider) randToken() string {
-	b := make([]byte, 32)
-	if _, err := rand.Read(b); err != nil {
-		log.Fatalf("[ERROR] can't get randoms, %s", err)
-	}
-	s := sha1.New()
-	if _, err := s.Write(b); err != nil {
-		log.Printf("[WARN] can't write randoms, %s", err)
-	}
-	return fmt.Sprintf("%x", s.Sum(nil))
-}

app/rest/auth/provider_test.go

@@ -1,218 +0,0 @@
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/http/cookiejar"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestLogin(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, permanent
-	resp, err := client.Get("http://localhost:8981/login?site=remark")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	t.Logf("resp %s", string(body))
-	t.Logf("headers: %+v", resp.Header)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 31536000, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	u := store.User{}
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser1", Picture: "http://exmple.com/pic1.png",
-		Admin: false, Blocked: false, IP: ""}, u)
-
-	// check admin user
-	resp, err = client.Get("http://localhost:8981/login?site=remark")
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser2", Picture: "http://exmple.com/pic1.png",
-		Admin: true, Blocked: false, IP: "", Verified: true}, u)
-}
-
-func TestLoginSessionOnly(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, session
-	resp, err := client.Get("http://localhost:8981/login?site=remark&session=1")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 0, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	req, err := http.NewRequest("GET", "http://example.com", nil)
-	require.Nil(t, err)
-	req.AddCookie(resp.Cookies()[0])
-	req.AddCookie(resp.Cookies()[1])
-	req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value)
-
-	jwtService := NewJWT("12345", false, time.Hour)
-	res, err := jwtService.Get(req)
-	require.Nil(t, err)
-	assert.Equal(t, true, res.SessionOnly)
-	t.Logf("%+v", res)
-}
-
-func TestLogout(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8691, 8692)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	resp, err := client.Get("http://localhost:8691/login")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	resp, err = client.Get("http://localhost:8691/logout")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name, "jwt cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[0].Value)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name, "xsrf cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[1].Value)
-}
-
-func TestInitProvider(t *testing.T) {
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret"}
-	provider := Provider{Name: "test", RedirectURL: "redir"}
-	res := initProvider(params, provider)
-	assert.Equal(t, "cid", res.conf.ClientID)
-	assert.Equal(t, "csecret", res.conf.ClientSecret)
-	assert.Equal(t, "redir", res.RedirectURL)
-	assert.Equal(t, "123456", res.SecretKey)
-	assert.Equal(t, "test", res.Name)
-}
-
-func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Server) {
-
-	provider := Provider{
-		Name: "mock",
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  fmt.Sprintf("http://localhost:%d/login/oauth/authorize", authPort),
-			TokenURL: fmt.Sprintf("http://localhost:%d/login/oauth/access_token", authPort),
-		},
-		RedirectURL: fmt.Sprintf("http://localhost:%d/callback", loginPort),
-		Scopes:      []string{"user:email"},
-		InfoURL:     fmt.Sprintf("http://localhost:%d/user", authPort),
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:      "mock_" + data.value("id"),
-				Name:    data.value("name"),
-				Picture: data.value("picture"),
-			}
-			return userInfo
-		},
-	}
-
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret",
-		JwtService: NewJWT("12345", false, time.Hour), Admins: []string{"mock_myuser2"},
-		// AvatarProxy:  &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"},
-		IsVerifiedFn: func(siteID, userID string) bool { return userID == "mock_myuser2" }}
-	provider = initProvider(params, provider)
-
-	ts := &http.Server{Addr: fmt.Sprintf(":%d", loginPort), Handler: provider.Routes()}
-
-	count := 0
-	useIds := []string{"myuser1", "myuser2"}
-
-	oauth := &http.Server{
-		Addr: fmt.Sprintf(":%d", authPort),
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			log.Printf("[MOCK OAUTH] request %s %s %+v", r.Method, r.URL, r.Header)
-			switch {
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
-				state := r.URL.Query().Get("state")
-				w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s",
-					loginPort, state))
-				w.WriteHeader(302)
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"):
-				res := `{
-					"access_token":"MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3",
-					"token_type":"bearer",
-					"expires_in":3600,
-					"refresh_token":"IwOGYzYTlmM2YxOTQ5MGE3YmNmMDFkNTVk",
-					"scope":"create",
-					"state":"12345678"
-					}`
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			case strings.HasPrefix(r.URL.Path, "/user"):
-				res := fmt.Sprintf(`{
-					"id": "%s",
-					"name":"blah",
-					"picture":"http://exmple.com/pic1.png"
-					}`, useIds[count])
-				count++
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			default:
-				t.Fatalf("unexpected oauth request %s %s", r.Method, r.URL)
-			}
-		}),
-	}
-
-	go func() { _ = oauth.ListenAndServe() }()
-	go func() { _ = ts.ListenAndServe() }()
-
-	time.Sleep(time.Millisecond * 100) // let them start
-	return ts, oauth
-}

app/rest/auth/providers.go

@@ -1,126 +0,0 @@
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"golang.org/x/oauth2/facebook"
-	"golang.org/x/oauth2/github"
-	"golang.org/x/oauth2/google"
-	"golang.org/x/oauth2/yandex"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// NewGoogle makes google oauth2 provider
-func NewGoogle(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "google",
-		Endpoint:    google.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/google/callback",
-		Scopes:      []string{"https://www.googleapis.com/auth/userinfo.profile"},
-		InfoURL:     "https://www.googleapis.com/oauth2/v3/userinfo",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				// encode email with provider name to avoid collision if same id returned by other provider
-				ID:      "google_" + store.EncodeID(data.value("sub")),
-				Name:    data.value("name"),
-				Picture: data.value("picture"),
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = "noname_" + userInfo.ID[8:12]
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewGithub makes github oauth2 provider
-func NewGithub(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "github",
-		Endpoint:    github.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/github/callback",
-		Scopes:      []string{},
-		InfoURL:     "https://api.github.com/user",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:      "github_" + store.EncodeID(data.value("login")),
-				Name:    data.value("name"),
-				Picture: data.value("avatar_url"),
-			}
-			// github may have no user name, use login in this case
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("login")
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewFacebook makes facebook oauth2 provider
-func NewFacebook(p Params) Provider {
-
-	// response format for fb /me call
-	type uinfo struct {
-		ID      string `json:"id"`
-		Name    string `json:"name"`
-		Picture struct {
-			Data struct {
-				URL string `json:"url"`
-			} `json:"data"`
-		} `json:"picture"`
-	}
-
-	return initProvider(p, Provider{
-		Name:        "facebook",
-		Endpoint:    facebook.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/facebook/callback",
-		Scopes:      []string{"public_profile"},
-		InfoURL:     "https://graph.facebook.com/me?fields=id,name,picture",
-		MapUser: func(data userData, bdata []byte) store.User {
-			userInfo := store.User{
-				ID:   "facebook_" + store.EncodeID(data.value("id")),
-				Name: data.value("name"),
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = userInfo.ID[0:16]
-			}
-
-			uinfoJSON := uinfo{}
-			if err := json.Unmarshal(bdata, &uinfoJSON); err == nil {
-				userInfo.Picture = uinfoJSON.Picture.Data.URL
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewYandex makes yandex oauth2 provider
-func NewYandex(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "yandex",
-		Endpoint:    yandex.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/yandex/callback",
-		Scopes:      []string{},
-		// See https://tech.yandex.com/passport/doc/dg/reference/response-docpage/
-		InfoURL: "https://login.yandex.ru/info?format=json",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:   "yandex_" + store.EncodeID(data.value("id")),
-				Name: data.value("display_name"), // using Display Name by default
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("real_name") // using Real Name (== full name) if Display Name is empty
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("login") // otherwise using login
-			}
-
-			if data.value("default_avatar_id") != "" {
-				userInfo.Picture = fmt.Sprintf("https://avatars.yandex.net/get-yapic/%s/islands-200", data.value("default_avatar_id"))
-			}
-			return userInfo
-		},
-	})
-}

app/rest/auth/providers_test.go

@@ -1,83 +0,0 @@
-package auth
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestProviders_NewGoogle(t *testing.T) {
-	r := NewGoogle(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "google", r.Name)
-
-	udata := userData{"sub": "1234567890", "name": "test user", "picture": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// no name in data
-	udata = userData{"sub": "1234567890", "picture": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "noname_1b30", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewGithub(t *testing.T) {
-	r := NewGithub(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "github", r.Name)
-
-	udata := userData{"login": "lll", "name": "test user", "avatar_url": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// nil name in data (json response contains `"name": null`); using login, it's always required
-	udata = userData{"login": "lll", "name": nil, "avatar_url": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "lll", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewFacebook(t *testing.T) {
-	r := NewFacebook(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "facebook", r.Name)
-
-	udata := userData{"id": "myid", "name": "test user"}
-	user := r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "test user", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	udata = userData{"id": "myid", "name": ""}
-	user = r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "facebook_6e34471", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewYandex(t *testing.T) {
-	r := NewYandex(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "yandex", r.Name)
-
-	udata := userData{"id": "1234567890", "display_name": "Vasya P", "default_avatar_id": "131652443"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya P", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "https://avatars.yandex.net/get-yapic/131652443/islands-200", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "display_name": null, "default_avatar_id": null
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": nil, "real_name": "Vasya Pupkin", "default_avatar_id": nil}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya Pupkin", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// empty "display_name", empty "default_avatar_id", empty "real_name"
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": "", "real_name": "", "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "real_name": null
-	udata = userData{"id": "1234567890", "login": "vasya", "real_name": nil, "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}

app/rest/cache/cache.go

@@ -1,45 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
-)
-
-// LoadingCache defines interface for caching
-type LoadingCache interface {
-	Get(key string, fn func() ([]byte, error)) (data []byte, err error)
-	Flush(scopes ...string)
-}
-
-// Key makes full key from primary key and scopes
-func Key(key string, scopes ...string) string {
-	return strings.Join(scopes, "$$") + "@@" + key
-}
-
-// ParseKey gets compound key created by Key func and split it to the actual key and scopes
-func ParseKey(fullKey string) (key string, scopes []string, err error) {
-	elems := strings.Split(fullKey, "@@")
-	if len(elems) != 2 {
-		return "", nil, errors.Errorf("can't parse cache key %s", key)
-	}
-	scopes = strings.Split(elems[0], "$$")
-	if len(scopes) == 1 && scopes[0] == "" {
-		scopes = []string{}
-	}
-	key = elems[1]
-	return key, scopes, nil
-}
-
-// URLKey gets url from request to use it as cache key
-// admins will have different keys in order to prevent leak of admin-only data to regular users
-func URLKey(r *http.Request) string {
-	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
-		key = adminPrefix + key
-	}
-	return key
-}

app/rest/cache/cache_test.go

@@ -1,54 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestCache_Keys(t *testing.T) {
-	tbl := []struct {
-		key    string
-		scopes []string
-		full   string
-	}{
-		{"key1", []string{"s1"}, "s1@@key1"},
-		{"key2", []string{"s11", "s2"}, "s11$$s2@@key2"},
-		{"key3", []string{}, "@@key3"},
-	}
-
-	for n, tt := range tbl {
-		full := Key(tt.key, tt.scopes...)
-		assert.Equal(t, tt.full, full, "making key, #%d", n)
-
-		k, s, e := ParseKey(full)
-		assert.Nil(t, e)
-		assert.Equal(t, tt.scopes, s)
-		assert.Equal(t, tt.key, k)
-	}
-
-	_, _, err := ParseKey("abc")
-	assert.Error(t, err)
-	_, _, err = ParseKey("")
-	assert.Error(t, err)
-}
-
-func TestCache_URLKey(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah/123", nil)
-	assert.Nil(t, err)
-	key := URLKey(r)
-	assert.Equal(t, "http://blah/123", key)
-
-	r, err = http.NewRequest("GET", "http://blah/123?key=v&k2=v2", nil)
-	assert.Nil(t, err)
-	key = URLKey(r)
-	assert.Equal(t, "http://blah/123?key=v&k2=v2", key)
-
-	user := store.User{Admin: true}
-	r = rest.SetUserInfo(r, user)
-	key = URLKey(r)
-	assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key)
-}

app/rest/cache/memory.go

@@ -1,114 +0,0 @@
-package cache
-
-import (
-	"log"
-	"sync/atomic"
-
-	"github.com/hashicorp/golang-lru"
-	"github.com/pkg/errors"
-)
-
-// memoryCache implements LoadingCache interface on top of cache.Cache (go-cache)
-type memoryCache struct {
-	bytesCache   *lru.Cache
-	postFlushFn  func()
-	maxKeys      int
-	maxValueSize int
-	maxCacheSize int64
-	currentSize  int64
-}
-
-// NewMemoryCache makes memoryCache implementation
-func NewMemoryCache(options ...Option) (LoadingCache, error) {
-	res := memoryCache{
-		postFlushFn:  func() {},
-		maxKeys:      1000,
-		maxValueSize: 0,
-	}
-	for _, opt := range options {
-		if err := opt(&res); err != nil {
-			return nil, errors.Wrap(err, "failed to set cache option")
-		}
-	}
-
-	onEvicted := func(key interface{}, value interface{}) {
-		size := len(value.([]byte))
-		atomic.AddInt64(&res.currentSize, -1*int64(size))
-	}
-
-	var err error
-	// OnEvicted called automatically for expired and manually deleted
-	if res.bytesCache, err = lru.NewWithEvict(res.maxKeys, onEvicted); err != nil {
-		return nil, errors.Wrap(err, "failed to make cache")
-	}
-
-	log.Printf("[DEBUG] create lru cache, maxKeys=%d, maxValueSize=%d, maxCacheSize=%d",
-		res.maxKeys, res.maxValueSize, res.maxCacheSize)
-	return &res, nil
-}
-
-// Get is loading cache method to get value by key or load via fn if not found
-func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
-	if b, ok := m.bytesCache.Get(key); ok {
-		return b.([]byte), nil
-	}
-
-	if data, err = fn(); err != nil {
-		return data, err
-	}
-	if m.allowed(data) {
-		m.bytesCache.Add(key, data)
-		atomic.AddInt64(&m.currentSize, int64(len(data)))
-
-		if m.maxCacheSize > 0 && atomic.LoadInt64(&m.currentSize) > m.maxCacheSize {
-			for atomic.LoadInt64(&m.currentSize) > m.maxCacheSize {
-				m.bytesCache.RemoveOldest()
-			}
-		}
-	}
-	return data, nil
-}
-
-// Flush clears cache and calls postFlushFn async
-func (m *memoryCache) Flush(scopes ...string) {
-
-	if len(scopes) == 0 {
-		m.bytesCache.Purge()
-		go m.postFlushFn()
-		return
-	}
-
-	// check if fullKey has matching scopes
-	inScope := func(fullKey string) bool {
-		_, keyScopes, err := ParseKey(fullKey)
-		if err != nil {
-			return false
-		}
-		for _, s := range scopes {
-			for _, ks := range keyScopes {
-				if ks == s {
-					return true
-				}
-			}
-		}
-		return false
-	}
-
-	for _, k := range m.bytesCache.Keys() {
-		key := k.(string)
-		if inScope(key) {
-			m.bytesCache.Remove(key) // Keys() returns copy of cache's key, safe to remove directly
-		}
-	}
-
-	if m.postFlushFn != nil {
-		go m.postFlushFn()
-	}
-}
-
-func (m *memoryCache) allowed(data []byte) bool {
-	if m.maxValueSize > 0 && len(data) >= m.maxValueSize {
-		return false
-	}
-	return true
-}

app/rest/cache/memory_test.go

@@ -1,310 +0,0 @@
-package cache
-
-import (
-	"fmt"
-	"math/rand"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMemoryCache_Get(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }))
-	require.Nil(t, err)
-	res, err := lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	res, err = lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	lc.Flush()
-	time.Sleep(100 * time.Millisecond) // let postFn to do its thing
-	assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall))
-
-	_, err = lc.Get("key", func() ([]byte, error) {
-		return nil, errors.New("err")
-	})
-	assert.NotNil(t, err)
-}
-
-func TestMemoryCache_MaxKeys(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }),
-		MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-
-	// put 5 keys to cache
-	for i := 0; i < 5; i++ {
-		res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-			atomic.AddInt32(&coldCalls, 1)
-			return []byte(fmt.Sprintf("result-%d", i)), nil
-		})
-		assert.Nil(t, e)
-		assert.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-		assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls))
-		assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-	}
-
-	// check if really cached
-	res, err := lc.Get("key-3", func() ([]byte, error) {
-		return []byte("result-blah"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-3", string(res), "should be cached")
-
-	// try to cache after maxKeys reached
-	res, err = lc.Get("key-X", func() ([]byte, error) {
-		return []byte("result-X"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-X", string(res))
-
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-
-	// put to cache and make sure it cached
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxValueSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-
-	// put too big value to cache and make sure it is not cached
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("1234567890"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "1234567890", string(res))
-
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("result-big"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-big", string(res), "got not cached value")
-}
-
-func TestMemoryCache_MaxCacheSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(50), MaxCacheSize(20))
-	require.Nil(t, err)
-
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-	assert.Equal(t, int64(8), lc.(*memoryCache).currentSize)
-
-	_, err = lc.Get("key-Z2", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	// this will cause removal
-	_, err = lc.Get("key-Z3", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) {
-	lc, err := NewMemoryCache(MaxCacheSize(123), MaxKeys(10000))
-	require.Nil(t, err)
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond)
-			defer wg.Done()
-			res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-			size := atomic.LoadInt64(&lc.(*memoryCache).currentSize)
-			require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel
-		}()
-	}
-	wg.Wait()
-	assert.True(t, lc.(*memoryCache).currentSize < 123 && lc.(*memoryCache).currentSize >= 0)
-	t.Log("size=", lc.(*memoryCache).currentSize)
-}
-
-func TestMemoryCache_Parallel(t *testing.T) {
-	var coldCalls int32
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get("key", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			defer wg.Done()
-			res, err := lc.Get("key", func() ([]byte, error) {
-				atomic.AddInt32(&coldCalls, 1)
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, "value", string(res))
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls))
-}
-
-func TestMemoryCache_Scopes(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		return []byte("value2"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value2", string(res))
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-	lc.Flush("s1")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	_, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		assert.Fail(t, "should stay")
-		return nil, nil
-	})
-	assert.Nil(t, err)
-	res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value-upd"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value-upd", string(res), "was deleted, update")
-}
-
-func TestMemoryCache_Flush(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	addToCache := func(key string, scopes ...string) {
-		res, err := lc.Get(key, func() ([]byte, error) {
-			return []byte("value" + key), nil
-		})
-		require.Nil(t, err)
-		require.Equal(t, "value"+key, string(res))
-	}
-
-	init := func() {
-		lc.Flush()
-		addToCache(Key("key1", "s1", "s2"))
-		addToCache(Key("key2", "s1", "s2", "s3"))
-		addToCache(Key("key3", "s1", "s2", "s3"))
-		addToCache(Key("key4", "s2", "s3"))
-		addToCache(Key("key5", "s2"))
-		addToCache(Key("key6"))
-		addToCache(Key("key7", "s4", "s3"))
-		require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init")
-	}
-
-	tbl := []struct {
-		scopes []string
-		left   int
-		msg    string
-	}{
-		{[]string{}, 0, "full flush, no scopes"},
-		{[]string{"s0"}, 7, "flush wrong scope"},
-		{[]string{"s1"}, 4, "flush s1 scope"},
-		{[]string{"s2", "s1"}, 2, "flush s2+s1 scope"},
-		{[]string{"s1", "s2"}, 2, "flush s1+s2 scope"},
-		{[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"},
-		{[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"},
-		{[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"},
-	}
-
-	for i, tt := range tbl {
-		init()
-		lc.Flush(tt.scopes...)
-		assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i)
-	}
-}
-
-func TestMemoryCache_FlushFailed(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-	val, err := lc.Get("invalid-composite", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(val))
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	lc.Flush("invalid-composite")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_BadOptions(t *testing.T) {
-	_, err := NewMemoryCache(MaxCacheSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size or MaxCacheSize, -1")
-
-	_, err = NewMemoryCache(MaxKeys(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxKeys, -1")
-
-	_, err = NewMemoryCache(MaxValSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxValSize, -1")
-}

app/rest/cache/options.go

@@ -1,50 +0,0 @@
-package cache
-
-import "github.com/pkg/errors"
-
-// Option func type
-type Option func(lc *memoryCache) error
-
-// MaxValSize functional option defines the largest value's size allowed to be cached
-// By default it is 0, which means unlimited.
-func MaxValSize(max int) Option {
-	return func(lc *memoryCache) error {
-		lc.maxValueSize = max
-		if max <= 0 {
-			return errors.Errorf("negative size for MaxValSize, %d", max)
-		}
-		return nil
-	}
-}
-
-// MaxKeys functional option defines how many keys to keep.
-// By default it is 0, which means unlimited.
-func MaxKeys(max int) Option {
-	return func(lc *memoryCache) error {
-		lc.maxKeys = max
-		if max <= 0 {
-			return errors.Errorf("negative size for MaxKeys, %d", max)
-		}
-		return nil
-	}
-}
-
-// MaxCacheSize functional option defines the total size of cached data.
-// By default it is 0, which means unlimited.
-func MaxCacheSize(max int64) Option {
-	return func(lc *memoryCache) error {
-		lc.maxCacheSize = max
-		if max <= 0 {
-			return errors.Errorf("negative size or MaxCacheSize, %d", max)
-		}
-		return nil
-	}
-}
-
-// PostFlushFn functional option defines how callback function called after each Flush.
-func PostFlushFn(postFlushFn func()) Option {
-	return func(lc *memoryCache) error {
-		lc.postFlushFn = postFlushFn
-		return nil
-	}
-}

app/rest/httperrors.go

@@ -1,44 +0,0 @@
-package rest
-
-import (
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"runtime"
-	"strings"
-
-	"github.com/go-chi/render"
-)
-
-// SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
-	render.Status(r, code)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
-}
-
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
-	uinfoStr := ""
-	if user, e := GetUserInfo(r); e == nil {
-		uinfoStr = user.Name + "/" + user.ID + " - "
-	}
-	q := r.URL.String()
-	if qun, e := url.QueryUnescape(q); e == nil {
-		q = qun
-	}
-
-	srcFileInfo := ""
-	if pc, file, line, ok := runtime.Caller(2); ok {
-		fnameElems := strings.Split(file, "/")
-		funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-		srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
-			line, funcNameElems[len(funcNameElems)-1])
-	}
-
-	remoteIP := r.RemoteAddr
-	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
-		remoteIP = remoteIP[:pos]
-	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
-}

app/rest/httperrors_test.go

@@ -1,60 +0,0 @@
-package rest
-
-import (
-	"errors"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestSendErrorJSON(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/error" {
-			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
-			return
-		}
-		w.WriteHeader(404)
-	}))
-
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/error")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
-	assert.Equal(t, 500, resp.StatusCode)
-
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
-}
-
-func TestErrorDetailsMsg(t *testing.T) {
-	callerFn := func() {
-		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		require.Nil(t, err)
-		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
-	}
-	callerFn()
-}
-
-func TestErrorDetailsMsgWithUser(t *testing.T) {
-	callerFn := func() {
-		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		req.RemoteAddr = "127.0.0.1:1234"
-		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
-		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
-	}
-	callerFn()
-}

app/rest/proxy/avatar.go

@@ -1,119 +0,0 @@
-package proxy
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Avatar provides file-system store and http handler for avatars
-// On user login auth will call Put and it will retrieve and save picture locally.
-type Avatar struct {
-	Store     AvatarStore
-	RoutePath string
-	RemarkURL string
-}
-
-const imgSfx = ".image"
-
-// Put stores retrieved avatar to StorePath. Gets image from user info. Returns proxied url
-func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
-
-	// no picture for user, try default avatar
-	if u.Picture == "" {
-		return "", errors.Errorf("no picture for %s", u.ID)
-	}
-
-	// load avatar from remote location
-	client := http.Client{Timeout: 10 * time.Second}
-	var resp *http.Response
-	err = retry(5, time.Second, func() error {
-		var e error
-		resp, e = client.Get(u.Picture)
-		return e
-	})
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch avatar from the orig")
-	}
-
-	defer func() {
-		if e := resp.Body.Close(); e != nil {
-			log.Printf("[WARN] can't close response body, %s", e)
-		}
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status)
-	}
-
-	avatar, err := p.Store.Put(u.ID, resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatar, u.Name)
-	return p.RemarkURL + p.RoutePath + "/" + avatar, nil
-}
-
-// Routes returns auth routes for given provider
-func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string, chi.Router) {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-
-	// GET /123456789.image
-	router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) {
-
-		avatar := chi.URLParam(r, "avatar")
-
-		// enforce client-side caching
-		etag := `"` + avatar + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-
-		avReader, size, err := p.Store.Get(avatar)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar")
-			return
-		}
-
-		defer func() {
-			if e := avReader.Close(); e != nil {
-				log.Printf("[WARN] can't close avatar reader for %s, %s", avatar, e)
-			}
-		}()
-
-		w.Header().Set("Content-Type", "image/*")
-		w.Header().Set("Content-Length", strconv.Itoa(size))
-		w.WriteHeader(http.StatusOK)
-		if _, err = io.Copy(w, avReader); err != nil {
-			log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
-		}
-	})
-
-	return p.RoutePath, router
-}
-
-func retry(retries int, delay time.Duration, fn func() error) (err error) {
-	for i := 0; i < retries; i++ {
-		if err = fn(); err == nil {
-			return nil
-		}
-		time.Sleep(delay)
-	}
-	return errors.Wrap(err, "retry failed")
-}

app/rest/proxy/avatar_store.go

@@ -1,145 +0,0 @@
-package proxy
-
-//go:generate sh -c "mockery -inpkg -name AvatarStore -print > /tmp/mock.tmp && mv /tmp/mock.tmp avatar_store_mock.go"
-
-import (
-	"bytes"
-	"fmt"
-	"hash/crc64"
-	"image"
-	"image/png"
-	"io"
-	"log"
-	"os"
-	"path"
-	"strings"
-	"sync"
-
-	// Initializing packages for supporting GIF and JPEG formats.
-	_ "image/gif"
-	_ "image/jpeg"
-
-	"github.com/pkg/errors"
-	"golang.org/x/image/draw"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// AvatarStore defines interface to store and serve avatars
-type AvatarStore interface {
-	Put(userID string, reader io.Reader) (avatar string, err error)
-	Get(avatar string) (reader io.ReadCloser, size int, err error)
-}
-
-// FSAvatarStore implements AvatarStore for local file system
-type FSAvatarStore struct {
-	storePath   string
-	resizeLimit int
-	ctcTable    *crc64.Table
-	once        sync.Once
-}
-
-// NewFSAvatarStore makes file-system avatar store
-func NewFSAvatarStore(storePath string, resizeLimit int) *FSAvatarStore {
-	return &FSAvatarStore{storePath: storePath, resizeLimit: resizeLimit}
-}
-
-// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
-func (fs *FSAvatarStore) Put(userID string, reader io.Reader) (avatar string, err error) {
-	id := store.EncodeID(userID)
-	location := fs.location(id) // location adds partition to path
-
-	if _, err = os.Stat(location); os.IsNotExist(err) {
-		if e := os.Mkdir(location, 0700); e != nil {
-			return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
-		}
-	}
-
-	avFile := path.Join(location, id+imgSfx)
-	fh, err := os.Create(avFile)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't create file %s", avFile)
-	}
-	defer func() {
-		if e := fh.Close(); e != nil {
-			log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
-		}
-	}()
-
-	// Trying to resize avatar.
-	if reader = resize(reader, fs.resizeLimit); reader == nil {
-		return "", errors.New("avatar reader is nil")
-	}
-
-	if _, err = io.Copy(fh, reader); err != nil {
-		return "", errors.Wrapf(err, "can't save file %s", avFile)
-	}
-	return id + imgSfx, nil
-}
-
-// Get avatar reader for avatar id.image
-func (fs *FSAvatarStore) Get(avatar string) (reader io.ReadCloser, size int, err error) {
-	location := fs.location(strings.TrimSuffix(avatar, imgSfx))
-	avFile := path.Join(location, avatar)
-	fh, err := os.Open(avFile)
-	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't load avatar %s, id")
-	}
-	if fi, e := fh.Stat(); e == nil {
-		size = int(fi.Size())
-	}
-	return fh, size, nil
-}
-
-// get location (directory) for user id by adding partition to final path in order to keep files
-// in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/avatars.test/92
-func (fs *FSAvatarStore) location(id string) string {
-	fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
-	checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
-	partition := checksum64 % 100
-	return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
-}
-
-// Resizes an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
-// (width or height) preserving aspect ratio.
-// Returns original reader if resizing is not needed or failed.
-func resize(reader io.Reader, limit int) io.Reader {
-	if reader == nil {
-		log.Print("[WARN] avatar resize(): reader is nil")
-		return nil
-	}
-	if limit <= 0 {
-		log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
-		return reader
-	}
-
-	var teeBuf bytes.Buffer
-	tee := io.TeeReader(reader, &teeBuf)
-	src, _, err := image.Decode(tee)
-	if err != nil {
-		log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
-		return &teeBuf
-	}
-
-	bounds := src.Bounds()
-	w, h := bounds.Dx(), bounds.Dy()
-	if w <= limit && h <= limit || w <= 0 || h <= 0 {
-		log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
-		return &teeBuf
-	}
-	newW, newH := w*limit/h, limit
-	if w > h {
-		newW, newH = limit, h*limit/w
-	}
-	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
-	// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
-	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
-
-	var out bytes.Buffer
-	if err = png.Encode(&out, m); err != nil {
-		log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
-		return &teeBuf
-	}
-	return &out
-}

app/rest/proxy/avatar_store_test.go

@@ -1,135 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"image"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAvatarStore_Put(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	avatar, err := p.Put("user1", nil)
-	assert.Equal(t, "", avatar)
-	assert.EqualError(t, err, "avatar reader is nil")
-
-	avatar, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	require.Nil(t, err)
-	assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	avatar, err = p.Put("user2", strings.NewReader("some picture bin data 123"))
-	require.Nil(t, err)
-	assert.Equal(t, "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(25), fi.Size())
-
-	// with resize
-	file, e := os.Open("testdata/circles.png")
-	require.Nil(t, e)
-	avatar, err = p.Put("user3", file)
-	require.Nil(t, err)
-	assert.Equal(t, "0b7f849446d3383546d15a480966084442cd2193.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/60/0b7f849446d3383546d15a480966084442cd2193.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(6986), fi.Size())
-
-	p = NewFSAvatarStore("/dev/null", 300)
-	_, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
-}
-
-func TestAvatarStore_Get(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test/30", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-	err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
-	assert.Nil(t, err)
-	r, size, err := p.Get("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.Nil(t, err)
-	assert.Equal(t, 9, size)
-	data, err := ioutil.ReadAll(r)
-	assert.Nil(t, err)
-	assert.Equal(t, "something", string(data))
-}
-
-func TestAvatarStore_Location(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-
-	tbl := []struct {
-		id  string
-		res string
-	}{
-		{"abc", "/tmp/avatars.test/35"},
-		{"xyz", "/tmp/avatars.test/69"},
-		{"blah blah", "/tmp/avatars.test/29"},
-	}
-
-	for i, tt := range tbl {
-		assert.Equal(t, tt.res, p.location(tt.id), "test #%d", i)
-	}
-}
-
-func TestAvatarStore_resize(t *testing.T) {
-	checkC := func(t *testing.T, r io.Reader, cExp []byte) {
-		content, err := ioutil.ReadAll(r)
-		require.NoError(t, err)
-		assert.Equal(t, cExp, content)
-	}
-
-	// Reader is nil.
-	resizedR := resize(nil, 100)
-	// assert.EqualError(t, err, "limit should be greater than 0")
-	assert.Nil(t, resizedR)
-
-	// Negative limit error.
-	resizedR = resize(strings.NewReader("some picture bin data"), -1)
-	require.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("some picture bin data"))
-
-	// Decode error.
-	resizedR = resize(strings.NewReader("invalid image content"), 100)
-	assert.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("invalid image content"))
-
-	cases := []struct {
-		file   string
-		wr, hr int
-	}{
-		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
-		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
-	}
-
-	for _, c := range cases {
-		img, err := ioutil.ReadFile(c.file)
-		require.Nil(t, err, "can't open test file %s", c.file)
-
-		// No need for resize, avatar dimensions are smaller than resize limit.
-		resizedR = resize(bytes.NewReader(img), 800)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-		checkC(t, resizedR, img)
-
-		// Resizing to half of width. Check resizedR avatar format PNG.
-		resizedR = resize(bytes.NewReader(img), 400)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-
-		imgRz, format, err := image.Decode(resizedR)
-		assert.Nilf(t, err, "file %s", c.file)
-		assert.Equalf(t, "png", format, "file %s", c.file)
-		bounds := imgRz.Bounds()
-		assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
-		assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
-	}
-}

app/rest/proxy/avatar_test.go

@@ -1,142 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestAvatar_Put(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	res, err := p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/b3daa77b4c04a9551b8781d03191fe098f325e67.image", res)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	u.ID = "user2"
-	res, err = p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", res)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-}
-
-func TestAvatar_PutFailed(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		log.Print("request: ", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-
-	u := store.User{ID: "user1", Name: "user1 name"}
-	_, err := p.Put(u)
-	assert.EqualError(t, err, "no picture for user1")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "connect: connection refused")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/avatar/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "failed to get avatar from the orig")
-}
-
-func TestAvatar_Routes(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			w.Header().Set("Custom-Header", "xyz")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	_, err := p.Put(u)
-	assert.NoError(t, err)
-
-	req, err := http.NewRequest("GET", "/b3daa77b4c04a9551b8781d03191fe098f325e67.image", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	rr := httptest.NewRecorder()
-	_, routes := p.Routes()
-	handler := http.Handler(routes)
-	handler.ServeHTTP(rr, req)
-
-	assert.Equal(t, http.StatusOK, rr.Code)
-
-	assert.Equal(t, []string{"image/*"}, rr.HeaderMap["Content-Type"])
-	assert.Equal(t, []string{"21"}, rr.HeaderMap["Content-Length"])
-	assert.Equal(t, []string(nil), rr.HeaderMap["Custom-Header"], "strip all custom headers")
-	assert.NotNil(t, rr.HeaderMap["Etag"])
-
-	bb := bytes.Buffer{}
-	sz, err := io.Copy(&bb, rr.Body)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), sz)
-	assert.Equal(t, "some picture bin data", bb.String())
-}
-
-func TestAvatar_Retry(t *testing.T) {
-	i := 0
-	err := retry(5, time.Millisecond, func() error {
-		if i == 3 {
-			return nil
-		}
-		i++
-		return errors.New("err")
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 3, i)
-
-	st := time.Now()
-	err = retry(5, time.Millisecond, func() error {
-		return errors.New("err")
-	})
-	assert.NotNil(t, err)
-	assert.True(t, time.Since(st) >= time.Microsecond*5)
-}

app/rest/proxy/image.go

@@ -1,126 +0,0 @@
-package proxy
-
-import (
-	"encoding/base64"
-	"io"
-	"log"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
-)
-
-// Image extracts image src from comment's html and provides proxy for them
-// this is needed to keep remark42 running behind of HTTPS serve all images via https
-type Image struct {
-	RemarkURL string
-	RoutePath string
-	Enabled   bool
-}
-
-// Convert all img src links without https to proxied links
-func (p Image) Convert(commentHTML string) string {
-	if !p.Enabled || strings.HasPrefix(p.RemarkURL, "http://") {
-		return commentHTML
-	}
-
-	imgs, err := p.extract(commentHTML)
-	if err != nil {
-		return commentHTML
-	}
-
-	return p.replace(commentHTML, imgs)
-}
-
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
-	if !p.Enabled {
-		return router
-	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
-			return
-		}
-
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = retry(5, time.Second, func() error {
-			var e error
-			resp, e = client.Get(string(src))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
-
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
-
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
-		}
-	})
-	return router
-}
-
-// extract gets all non-https images and return list of src
-func (p Image) extract(commentHTML string) ([]string, error) {
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
-	if err != nil {
-		return nil, errors.Wrap(err, "can't create document")
-	}
-	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
-		if im, ok := s.Attr("src"); ok {
-			if strings.HasPrefix(im, "http://") {
-				result = append(result, im)
-			}
-		}
-	})
-	return result, nil
-}
-
-// replace img links in commentHTML with route to proxy with base64 encoded original link
-func (p Image) replace(commentHTML string, imgs []string) string {
-
-	for _, img := range imgs {
-		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
-		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
-		commentHTML = strings.Replace(commentHTML, img, resImgURL, -1)
-	}
-
-	return commentHTML
-}

app/rest/proxy/image_test.go

@@ -1,121 +0,0 @@
-package proxy
-
-import (
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestPicture_Extract(t *testing.T) {
-
-	tbl := []struct {
-		inp string
-		res []string
-	}{
-		{
-			`<p> blah <img src="http://radio-t.com/img.png"/> test</p>`,
-			[]string{"http://radio-t.com/img.png"},
-		},
-		{
-			`<p> blah <img src="https://radio-t.com/img.png"/> test</p>`,
-			[]string{},
-		},
-		{
-			`<img src="http://radio-t.com/img2.png"/>`,
-			[]string{"http://radio-t.com/img2.png"},
-		},
-		{
-			`<img src="http://radio-t.com/img3.png"/> <div>xyz <img src="http://images.pexels.com/67636/img4.jpeg"> </div>`,
-			[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"},
-		},
-		{
-			`<img src="https://radio-t.com/img3.png"/> <div>xyz <img src="http://images.pexels.com/67636/img4.jpeg"> </div>`,
-			[]string{"http://images.pexels.com/67636/img4.jpeg"},
-		},
-		{
-			`abcd <b>blah</b> <h1>xxx</h1>`,
-			[]string{},
-		},
-	}
-	img := Image{Enabled: true}
-
-	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
-	}
-}
-
-func TestPicture_Replace(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
-	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
-		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
-	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-}
-
-func TestImage_Routes(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
-
-	httpSrv := imgHTTPServer(t)
-	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
-
-	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	t.Logf("%+v", resp.Header)
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
-
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
-
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
-}
-
-func TestPicture_Convert(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
-	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
-	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-
-	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
-	assert.Equal(t, `<img src="https://radio-t.com/img3.png"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-
-	img = Image{Enabled: true, RoutePath: "/img", RemarkURL: "http://example.com"}
-	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
-	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "http:// remark url, no proxy")
-
-	img = Image{Enabled: false, RoutePath: "/img"}
-	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
-	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
-}
-
-func imgHTTPServer(t *testing.T) *httptest.Server {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/image/img1.png" {
-			t.Log("http img request", r.URL)
-			w.Header().Add("Content-Length", "123")
-			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
-			return
-		}
-		t.Log("http img request - not found", r.URL)
-		w.WriteHeader(404)
-	}))
-
-	return ts
-}

app/rest/tree_test.go

@@ -1,396 +0,0 @@
-package rest
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestMakeTree(t *testing.T) {
-
-	loc := store.Locator{URL: "url", SiteID: "site"}
-	ts := func(min int, sec int) time.Time { return time.Date(2017, 12, 25, 19, min, sec, 0, time.UTC) }
-
-	// unsorted by purpose
-	comments := []store.Comment{
-		{Locator: loc, ID: "14", ParentID: "1", Timestamp: ts(46, 14)},
-		{Locator: loc, ID: "1", Timestamp: ts(46, 1)},
-		{Locator: loc, ID: "2", Timestamp: ts(47, 2)},
-		{Locator: loc, ID: "11", ParentID: "1", Timestamp: ts(46, 11)},
-		{Locator: loc, ID: "13", ParentID: "1", Timestamp: ts(46, 13)},
-		{Locator: loc, ID: "12", ParentID: "1", Timestamp: ts(46, 12)},
-		{Locator: loc, ID: "131", ParentID: "13", Timestamp: ts(46, 31)},
-		{Locator: loc, ID: "132", ParentID: "13", Timestamp: ts(46, 32)},
-		{Locator: loc, ID: "21", ParentID: "2", Timestamp: ts(47, 21)},
-		{Locator: loc, ID: "22", ParentID: "2", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "4", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "3", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "5", Deleted: true},
-		{Locator: loc, ID: "6", Deleted: true},
-		{Locator: loc, ID: "61", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "62", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
-	}
-
-	res := MakeTree(comments, "time", 0)
-
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(res)
-	assert.Nil(t, err)
-	expected, actual := cleanFormatting(expJSON, buf.String())
-	assert.Equal(t, expected, actual)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
-
-	res = MakeTree([]store.Comment{}, "time", 0)
-	assert.Equal(t, &Tree{}, res)
-
-	res = MakeTree(comments, "time", 10)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
-}
-
-func TestTreeSortNodes(t *testing.T) {
-	// unsorted by purpose
-	comments := []store.Comment{
-		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
-		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
-		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
-		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
-		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
-		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
-		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
-		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
-		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
-	}
-
-	res := MakeTree(comments, "+active", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-
-	res = MakeTree(comments, "-active", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "+time", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-time", 0)
-	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "+score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-score", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "undefined", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-}
-
-func BenchmarkTree(b *testing.B) {
-	comments := []store.Comment{}
-	data, err := ioutil.ReadFile("testfile.json")
-	assert.Nil(b, err)
-	err = json.Unmarshal(data, &comments)
-	assert.Nil(b, err)
-
-	for i := 0; i < b.N; i++ {
-		res := MakeTree(comments, "time", 0)
-		assert.NotNil(b, res)
-	}
-}
-
-const expJSON = `{
-  "comments": [
-    {
-      "comment": {
-        "id": "1",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": { 
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:46:01Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "11",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-          	  "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:11Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "12",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:12Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "13",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:13Z"
-          },
-          "replies": [
-            {
-              "comment": {
-                "id": "131",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:31Z"
-              }
-            },
-            {
-              "comment": {
-                "id": "132",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:32Z"
-              }
-            }
-          ]
-        },
-        {
-          "comment": {
-            "id": "14",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:14Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "2",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:02Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "21",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:21Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "22",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:22Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "4",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    },
-    {
-      "comment": {
-        "id": "3",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    }
-  ],
-  "info": {
-	 "url": "url",
-	 "count": 12,
-     "first_time": "2017-12-25T19:46:01Z",
-     "last_time": "2017-12-25T19:47:22Z"
-   }
-}
-`
-
-func cleanFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/user.go

@@ -1,32 +0,0 @@
-package rest
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/umputun/remark/app/store"
-)
-
-type contextKey string
-
-// GetUserInfo returns user from request context
-func GetUserInfo(r *http.Request) (user store.User, err error) {
-
-	ctx := r.Context()
-	if ctx == nil {
-		return store.User{}, errors.New("no info about user")
-	}
-	if u, ok := ctx.Value(contextKey("user")).(store.User); ok {
-		return u, nil
-	}
-
-	return store.User{}, errors.New("user can't be parsed")
-}
-
-// SetUserInfo sets user into request context
-func SetUserInfo(r *http.Request, user store.User) *http.Request {
-	ctx := r.Context()
-	ctx = context.WithValue(ctx, contextKey("user"), user)
-	return r.WithContext(ctx)
-}

app/rest/user_test.go

@@ -1,21 +0,0 @@
-package rest
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestGetUserInfo(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
-	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
-
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
-	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
-}

app/store/comment.go

@@ -1,103 +0,0 @@
-package store
-
-import (
-	"html/template"
-	"regexp"
-	"time"
-
-	"github.com/microcosm-cc/bluemonday"
-)
-
-// Comment represents a single comment with optional reference to its parent
-type Comment struct {
-	ID        string          `json:"id"`
-	ParentID  string          `json:"pid"`
-	Text      string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
-	Locator   Locator         `json:"locator"`
-	Score     int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
-	Timestamp time.Time       `json:"time"`
-	Edit      *Edit           `json:"edit,omitempty"` // pointer to have empty default in json response
-	Pin       bool            `json:"pin,omitempty"`
-	Deleted   bool            `json:"delete,omitempty"`
-}
-
-// Locator keeps site and url of the post
-type Locator struct {
-	SiteID string `json:"site,omitempty"`
-	URL    string `json:"url"`
-}
-
-// Edit indication
-type Edit struct {
-	Timestamp time.Time `json:"time"`
-	Summary   string    `json:"summary"`
-}
-
-// PostInfo holds summary for given post url
-type PostInfo struct {
-	URL      string    `json:"url"`
-	Count    int       `json:"count"`
-	ReadOnly bool      `json:"read_only,omitempty"`
-	FirstTS  time.Time `json:"first_time,omitempty"`
-	LastTS   time.Time `json:"last_time,omitempty"`
-}
-
-// BlockedUser holds id and ts for blocked user
-type BlockedUser struct {
-	ID        string    `json:"id"`
-	Name      string    `json:"name"`
-	Timestamp time.Time `json:"time"`
-}
-
-// DeleteMode defines how much comment info will be erased
-type DeleteMode int
-
-// DeleteMode enum
-const (
-	SoftDelete DeleteMode = 0
-	HardDelete DeleteMode = 1
-)
-
-// PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
-// autogen fields and reset everything users not supposed to provide
-func (c *Comment) PrepareUntrusted() {
-	c.ID = ""                 // don't allow user to define ID, force auto-gen
-	c.Timestamp = time.Time{} // reset time, force auto-gen
-	c.Votes = make(map[string]bool)
-	c.Score = 0
-	c.Edit = nil
-	c.Pin = false
-	c.Deleted = false
-}
-
-// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well
-func (c *Comment) SetDeleted(mode DeleteMode) {
-	c.Text = ""
-	c.Orig = ""
-	c.Score = 0
-	c.Votes = map[string]bool{}
-	c.Edit = nil
-	c.Deleted = true
-	c.Pin = false
-
-	if mode == HardDelete {
-		c.User.Name = "deleted"
-		c.User.ID = "deleted"
-		c.User.Picture = ""
-		c.User.IP = ""
-	}
-}
-
-// Sanitize clean dangerous html/js from the comment
-func (c *Comment) Sanitize() {
-	p := bluemonday.UGCPolicy()
-	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
-	c.Text = p.Sanitize(c.Text)
-	c.Orig = p.Sanitize(c.Orig)
-	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = template.HTMLEscapeString(c.User.Name)
-	c.User.Picture = p.Sanitize(c.User.Picture)
-}

app/store/comment_test.go

@@ -1,112 +0,0 @@
-package store
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestComment_Sanitize(t *testing.T) {
-
-	tbl := []struct {
-		inp Comment
-		out Comment
-	}{
-		{inp: Comment{}, out: Comment{}},
-		{
-			inp: Comment{
-				Text: `blah <a href="javascript:alert('XSS1')" onmouseover="alert('XSS2')">XSS<a>` + "\n\t",
-				User: User{ID: `<a href="http://blah.com">username</a>`},
-			},
-			out: Comment{
-				Text: "blah XSS\n\t",
-				User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
-			},
-		},
-	}
-
-	for n, tt := range tbl {
-		tt.inp.Sanitize()
-		assert.Equal(t, tt.out, tt.inp, "check #%d", n)
-	}
-}
-
-func TestComment_PrepareUntrusted(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "username"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Pin:       true,
-		Deleted:   true,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-	}
-
-	comment.PrepareUntrusted()
-	assert.Equal(t, "", comment.ID)
-	assert.Equal(t, "p123", comment.ParentID)
-	assert.Equal(t, "blah", comment.Text)
-	assert.Equal(t, 0, comment.Score)
-	assert.Equal(t, false, comment.Pin)
-	assert.Equal(t, time.Time{}, comment.Timestamp)
-	assert.Equal(t, false, comment.Deleted)
-	assert.Equal(t, make(map[string]bool), comment.Votes)
-	assert.Equal(t, User{ID: "username"}, comment.User)
-
-}
-
-func TestComment_SetDeleted(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Deleted:   false,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-		Pin:       true,
-	}
-
-	comment.SetDeleted(SoftDelete)
-
-	assert.Equal(t, "", comment.Text)
-	assert.Equal(t, "", comment.Orig)
-	assert.Equal(t, map[string]bool{}, comment.Votes)
-	assert.Equal(t, 0, comment.Score)
-	assert.True(t, comment.Deleted)
-	assert.Nil(t, comment.Edit)
-	assert.False(t, comment.Pin)
-	assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User)
-}
-
-func TestComment_SetDeletedHard(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Deleted:   false,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-		Pin:       true,
-	}
-
-	comment.SetDeleted(HardDelete)
-
-	assert.Equal(t, "", comment.Text)
-	assert.Equal(t, "", comment.Orig)
-	assert.Equal(t, map[string]bool{}, comment.Votes)
-	assert.Equal(t, 0, comment.Score)
-	assert.True(t, comment.Deleted)
-	assert.Nil(t, comment.Edit)
-	assert.False(t, comment.Pin)
-	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
-}

app/store/engine/bolt_accessor.go

@@ -1,530 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
-	dbs map[string]*bolt.DB
-}
-
-const (
-	// top level buckets
-	postsBucketName    = "posts"
-	lastBucketName     = "last"
-	userBucketName     = "users"
-	blocksBucketName   = "block"
-	infoBucketName     = "info"
-	readonlyBucketName = "readonly"
-	verifiedBucketName = "verified"
-
-	// limits
-	lastLimit = 1000
-	userLimit = 500
-)
-
-const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-
-// BoltSite defines single site param
-type BoltSite struct {
-	FileName string // full path to boltdb
-	SiteID   string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
-	log.Printf("[INFO] bolt store for sites %+v", sites)
-	result := BoltDB{dbs: make(map[string]*bolt.DB)}
-	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
-		}
-
-		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
-			infoBucketName, readonlyBucketName, verifiedBucketName}
-		err = db.Update(func(tx *bolt.Tx) error {
-			for _, bktName := range topBuckets {
-				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-					return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-				}
-			}
-			return nil
-		})
-
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create top level bucket)")
-		}
-
-		result.dbs[site.SiteID] = db
-	}
-	return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
-	bdb, err := b.db(comment.Locator.SiteID)
-	if err != nil {
-		return "", err
-	}
-
-	if b.IsReadOnly(comment.Locator) {
-		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
-	}
-
-	err = bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
-		if e != nil {
-			return e
-		}
-
-		// check if key already in store, reject doubles
-		if postBkt.Get([]byte(comment.ID)) != nil {
-			return errors.Errorf("key %s already in store", comment.ID)
-		}
-
-		// serialize comment to json []byte for bolt and save
-		if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
-			return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
-		}
-
-		ref := b.makeRef(comment)
-
-		// add reference to comment to "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		e = lastBkt.Put(commentTs, ref)
-		if e != nil {
-			return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
-		}
-
-		// add reference to commentID to "users" bucket
-		userBkt, e := b.getUserBucket(tx, comment.User.ID)
-		if e != nil {
-			return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
-		}
-		// put into individual user's bucket with ts as a key
-		if e = userBkt.Put(commentTs, ref); e != nil {
-			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
-		}
-
-		// set info with countfor post url
-		if _, e = b.setInfo(tx, comment); e != nil {
-			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
-		}
-		return nil
-	})
-
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		return bucket.ForEach(func(k, v []byte) error {
-			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
-				return errors.Wrap(e, "failed to unmarshal")
-			}
-			comments = append(comments, comment)
-			return nil
-		})
-	})
-
-	comments = sortComments(comments, sortFld)
-	return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
-
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		c := lastBkt.Cursor()
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			url, commentID, e := b.parseRef(v)
-			if e != nil {
-				return e
-			}
-			postBkt, e := b.getPostBucket(tx, url)
-			if e != nil {
-				return e
-			}
-
-			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
-				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
-				continue
-			}
-			if comment.Deleted {
-				continue
-			}
-			comments = append(comments, comment)
-			if len(comments) >= max {
-				break
-			}
-		}
-		return nil
-	})
-
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return 0, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		var e error
-		count, e = b.count(tx, locator.URL, 0)
-		return e
-	})
-
-	return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		postsBkt := tx.Bucket([]byte(postsBucketName))
-
-		c := postsBkt.Cursor()
-		n := 0
-		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			n++
-			if skip > 0 && n <= skip {
-				continue
-			}
-			postURL := string(k)
-			infoBkt := tx.Bucket([]byte(infoBucketName))
-			info := store.PostInfo{}
-			if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
-				return errors.Wrapf(e, "can't load info for %s", postURL)
-			}
-			list = append(list, info)
-			if limit > 0 && len(list) >= limit {
-				break
-			}
-		}
-		return nil
-	})
-
-	return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return store.PostInfo{}, err
-	}
-
-	info := store.PostInfo{}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		infoBkt := tx.Bucket([]byte(infoBucketName))
-		if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
-			return errors.Wrapf(e, "can't load info for %s", locator.URL)
-		}
-		return nil
-	})
-
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if b.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-	commentRefs := []string{}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	if limit == 0 || limit > userLimit {
-		limit = userLimit
-	}
-
-	// get list of references to comments
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-
-		c := userIDBkt.Cursor()
-		skipComments := 0
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			if len(commentRefs) >= limit {
-				break
-			}
-			if skip > 0 && skipComments < skip {
-				skipComments++
-				continue
-			}
-			commentRefs = append(commentRefs, string(v))
-		}
-		return nil
-	})
-
-	if err != nil {
-		return comments, err
-	}
-
-	// retrieve comments for refs
-	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
-		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
-			comments = append(comments, c)
-		}
-	}
-
-	return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return 0, err
-	}
-	count := 0
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-		stats := userIDBkt.Stats()
-		count = stats.KeyN
-		return nil
-	})
-	return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return comment, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.load(bucket, []byte(commentID), &comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
-	if curComment, err := b.Get(locator, comment.ID); err == nil {
-		// preserve immutable fields
-		comment.ParentID = curComment.ParentID
-		comment.Locator = curComment.Locator
-		comment.Timestamp = curComment.Timestamp
-		comment.User = curComment.User
-	}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.save(bucket, []byte(comment.ID), comment)
-	})
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res := postsBkt.Bucket([]byte(postURL))
-	if res == nil {
-		return nil, errors.Errorf("no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
-	if err != nil {
-		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
-	usersBkt := tx.Bucket([]byte(userBucketName))
-	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
-	if e != nil {
-		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
-	}
-	return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
-	if value == nil {
-		return errors.Errorf("can't save nil value for %s", key)
-	}
-	jdata, jerr := json.Marshal(value)
-	if jerr != nil {
-		return errors.Wrap(jerr, "can't marshal comment")
-	}
-	if err = bkt.Put(key, jdata); err != nil {
-		return errors.Wrapf(err, "failed to save key %s", key)
-	}
-	return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
-	value := bkt.Get(key)
-	if value == nil {
-		return errors.Errorf("no value for %s", key)
-	}
-
-	if err := json.Unmarshal(value, &res); err != nil {
-		return errors.Wrap(err, "failed to unmarshal")
-	}
-	return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
-		info = store.PostInfo{}
-	}
-	if val == 0 { // get current count, don't update
-		return info.Count, nil
-	}
-	info.Count += val
-
-	return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
-		info = store.PostInfo{
-			Count:   0,
-			URL:     comment.Locator.URL,
-			FirstTS: comment.Timestamp,
-			LastTS:  comment.Timestamp,
-		}
-	}
-	info.Count++
-	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
-	if res, ok := b.dbs[siteID]; ok {
-		return res, nil
-	}
-	return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
-	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
-	elems := strings.Split(string(val), "!!")
-	if len(elems) != 2 {
-		return "", "", errors.Errorf("invalid reference value %s", string(val))
-	}
-	return elems[0], elems[1], nil
-}

app/store/engine/bolt_accessor_test.go

@@ -1,372 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
-	assert.Equal(t, "key id-1 already in store", err.Error())
-
-	_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	comment := store.Comment{
-		ID:        "id-ro",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	err := b.SetReadOnly(comment.Locator, true)
-	require.Nil(t, err)
-
-	_, err = b.Create(comment)
-	assert.NotNil(t, err)
-	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
-	err = b.SetReadOnly(comment.Locator, false)
-	require.Nil(t, err)
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err)
-
-	_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = b.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = b.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = b.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	_, err = b.Last("bad", 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-
-	_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
-	res, err := b.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = b.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = b.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = b.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	_, err = b.List("bad", 1, 1)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	_, err = b.User("bad", "user1", 1, 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.User("radio-t", "userZ", 1, 0)
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
-	os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	defer os.Remove(testDb)
-
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err = b.Create(c)
-		require.Nil(t, err)
-	}
-
-	// get all comments
-	res, err := b.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = b.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	count, err := b.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	_, err = b.UserCount("bad", "user1")
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.UserCount("radio-t", "userZ")
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
-	b := BoltDB{}
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	res := b.makeRef(comment)
-	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
-	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/2", url)
-	assert.Equal(t, "12345", id)
-
-	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
-	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
-	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
-	os.Remove(testDb)
-
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b := boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/store/engine/bolt_admin.go

@@ -1,305 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"log"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
-			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
-		}
-		// set deleted status and clear fields
-		comment.SetDeleted(mode)
-
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
-			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
-		}
-
-		// delete from "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
-			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
-		}
-
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
-		return nil
-	})
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
-	// delete top-level buckets
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		for _, bktName := range toDelete {
-
-			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
-			}
-			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-			}
-		}
-		return nil
-	})
-
-	return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// get list of all comments outside of transaction loop
-	posts, err := b.List(siteID, 0, 0)
-	if err != nil {
-		return err
-	}
-
-	type commentInfo struct {
-		locator   store.Locator
-		commentID string
-	}
-
-	// get list of commentID for all user's comment
-	comments := []commentInfo{}
-	for _, postInfo := range posts {
-		err = bdb.View(func(tx *bolt.Tx) error {
-			postsBkt := tx.Bucket([]byte(postsBucketName))
-			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
-				comment := store.Comment{}
-				if err = json.Unmarshal(commentVal, &comment); err != nil {
-					return errors.Wrap(err, "failed to unmarshal")
-				}
-				if comment.User.ID == userID {
-					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
-				}
-				return nil
-			})
-			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
-	// delete collected comments
-	for _, ci := range comments {
-		if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
-			return errors.Wrapf(err, "failed to delete comment %+v", ci)
-		}
-	}
-
-	//  delete  user bucket
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		if usersBkt != nil {
-			if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
-				return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "can't delete user meta")
-	}
-
-	if len(comments) == 0 {
-		return errors.Errorf("unknown user %s", userID)
-	}
-
-	return err
-}
-
-// SetBlock blocks/unblocks user for given site
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
-			}
-		}
-		return nil
-	})
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		blocked = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
-			}
-
-			// get user name from comment user section
-			userName := ""
-			userComments, e := b.User(siteID, string(k), 1, 0)
-			if e == nil && len(userComments) > 0 {
-				userName = userComments[0].User.Name
-			}
-			users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
-			return nil
-		})
-	})
-
-	return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
-			}
-		case false:
-			if e := bucket.Delete([]byte(locator.URL)); e != nil {
-				return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
-			}
-		}
-		return nil
-	})
-}
-
-// IsReadOnly checks if user blocked
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		ro = bucket.Get([]byte(locator.URL)) != nil
-		return nil
-	})
-	return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set verified status for %s", userID)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean verified status for %s", userID)
-			}
-		}
-		return nil
-	})
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		verified = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return verified
-}

app/store/engine/bolt_admin_test.go

@@ -1,200 +0,0 @@
-package engine
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	err = b.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	err = b.DeleteAll("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	err := b.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	_, err = b.User("radio-t", "user1", 5, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	err = b.DeleteUser("radio-t-bad", "user1")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", false))
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetBlock("radio-t", "userX", false))
-
-	assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true))
-	assert.NoError(t, b.SetBlock("radio-t", "user2", true))
-	assert.NoError(t, b.SetBlock("radio-t", "user3", false))
-
-	ids, err := b.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	_, err = b.Blocked("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, b.SetVerified("radio-t", "u1", false))
-	assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-}

app/store/engine/engine.go

@@ -1,83 +0,0 @@
-// Package engine defines interfaces each supported storage should implement.
-// Includes default implementation with boltdb
-package engine
-
-import (
-	"sort"
-	"strings"
-
-	"github.com/umputun/remark/app/store"
-)
-
-//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go"
-
-// Interface combines all store interfaces
-type Interface interface {
-	Accessor
-	Admin
-}
-
-// UserRequest is the request send to get comments by user
-type UserRequest struct {
-	SiteID string
-	UserID string
-	Limit  int
-	Skip   int
-}
-
-// Accessor defines all usual access ops avail for regular user
-type Accessor interface {
-	Create(comment store.Comment) (commentID string, err error)           // create new comment, avoid dups by id
-	Get(locator store.Locator, commentID string) (store.Comment, error)   // get comment by id
-	Put(locator store.Locator, comment store.Comment) error               // update comment, mutable parts only
-	Find(locator store.Locator, sort string) ([]store.Comment, error)     // find comments for locator
-	Last(siteID string, limit int) ([]store.Comment, error)               // last comments for given site, sorted by time
-	User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
-	UserCount(siteID, userID string) (int, error)                         // comments count by user
-	Count(locator store.Locator) (int, error)                             // number of comments for the post
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)    // list of commented posts
-	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)  // get post info
-}
-
-// Admin defines all store ops avail for admin only
-type Admin interface {
-	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
-	DeleteAll(siteID string) error                                               // delete all data from site
-	DeleteUser(siteID string, userID string) error                               // remove all comments from user
-	SetBlock(siteID string, userID string, status bool) error                    // block or unblock  user
-	IsBlocked(siteID string, userID string) bool                                 // check if user blocked
-	Blocked(siteID string) ([]store.BlockedUser, error)                          // get list of blocked users
-	SetReadOnly(locator store.Locator, status bool) error                        // set/reset read-only flag
-	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
-	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
-	IsVerified(siteID string, userID string) bool                                // check verified status
-}
-
-// sortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
-	sort.Slice(comments, func(i, j int) bool {
-		switch sortFld {
-		case "+time", "-time", "time", "+active", "-active", "active":
-			if strings.HasPrefix(sortFld, "-") {
-				return comments[i].Timestamp.After(comments[j].Timestamp)
-			}
-			return comments[i].Timestamp.Before(comments[j].Timestamp)
-
-		case "+score", "-score", "score":
-			if strings.HasPrefix(sortFld, "-") {
-				if comments[i].Score == comments[j].Score {
-					return comments[i].Timestamp.Before(comments[j].Timestamp)
-				}
-				return comments[i].Score > comments[j].Score
-			}
-			if comments[i].Score == comments[j].Score {
-				return comments[i].Timestamp.Before(comments[j].Timestamp)
-			}
-			return comments[i].Score < comments[j].Score
-
-		default:
-			return comments[i].Timestamp.Before(comments[j].Timestamp)
-		}
-	})
-	return comments
-}

app/store/engine/engine_test.go

@@ -1,43 +0,0 @@
-package engine
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestEngine_sortComments(t *testing.T) {
-	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
-	}
-
-	sortComments(cc, "+time")
-	assert.Equal(t, "1", cc[0].ID)
-	assert.Equal(t, "2", cc[1].ID)
-	assert.Equal(t, "3", cc[2].ID)
-	assert.Equal(t, "4", cc[3].ID)
-
-	sortComments(cc, "-time")
-	assert.Equal(t, "4", cc[0].ID)
-	assert.Equal(t, "3", cc[1].ID)
-	assert.Equal(t, "2", cc[2].ID)
-	assert.Equal(t, "1", cc[3].ID)
-
-	sortComments(cc, "score")
-	assert.Equal(t, "2", cc[0].ID)
-	assert.Equal(t, "1", cc[1].ID)
-	assert.Equal(t, "3", cc[2].ID)
-	assert.Equal(t, "4", cc[3].ID)
-
-	sortComments(cc, "-score")
-	assert.Equal(t, "3", cc[0].ID)
-	assert.Equal(t, "4", cc[1].ID)
-	assert.Equal(t, "1", cc[2].ID)
-	assert.Equal(t, "2", cc[3].ID)
-}

app/store/service/service.go

@@ -1,189 +0,0 @@
-package service
-
-import (
-	"sync"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-)
-
-// DataStore wraps store.Interface with additional methods
-type DataStore struct {
-	engine.Interface
-	EditDuration   time.Duration
-	Secret         string
-	MaxCommentSize int
-
-	// granular locks
-	scopedLocks struct {
-		sync.Mutex
-		sync.Once
-		locks map[string]sync.Locker
-	}
-}
-
-const defaultCommentMaxSize = 2000
-
-// Create prepares comment and forward to Interface.Create
-func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
-	// fill ID and time if empty
-	if comment.ID == "" {
-		comment.ID = uuid.New().String()
-	}
-	if comment.Timestamp.IsZero() {
-		comment.Timestamp = time.Now()
-	}
-	// reset votes if nothing
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-
-	comment.Sanitize()            // clear potentially dangerous js from all parts of comment
-	comment.User.HashIP(s.Secret) // replace ip by hash
-
-	return s.Interface.Create(comment)
-}
-
-// SetPin pin/un-pin comment as special
-func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
-	comment, err := s.Get(locator, commentID)
-	if err != nil {
-		return err
-	}
-	comment.Pin = status
-	return s.Put(locator, comment)
-}
-
-// Vote for comment by id and locator
-func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
-
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
-	defer cLock.Unlock()
-
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	if comment.User.ID == userID && userID != "dev" {
-		return comment, errors.Errorf("user %s can not vote for his own comment %s", userID, commentID)
-	}
-
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-	v, voted := comment.Votes[userID]
-
-	if voted && v == val {
-		return comment, errors.Errorf("user %s already voted for %s", userID, commentID)
-	}
-
-	// reset vote if user changed to opposite
-	if voted && v != val {
-		delete(comment.Votes, userID)
-	}
-
-	// add to voted map if first vote
-	if !voted {
-		comment.Votes[userID] = val
-	}
-
-	// update score
-	if val {
-		comment.Score++
-	} else {
-		comment.Score--
-	}
-
-	return comment, s.Put(locator, comment)
-}
-
-// EditRequest contains fields needed for comment update
-type EditRequest struct {
-	Text    string
-	Orig    string
-	Summary string
-}
-
-// EditComment to edit text and update Edit info
-func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	// edit allowed in editDuration window only
-	if s.EditDuration > 0 && time.Now().After(comment.Timestamp.Add(s.EditDuration)) {
-		return comment, errors.Errorf("too late to edit %s", commentID)
-	}
-
-	comment.Text = req.Text
-	comment.Orig = req.Orig
-	comment.Edit = &store.Edit{
-		Timestamp: time.Now(),
-		Summary:   req.Summary,
-	}
-
-	comment.Sanitize()
-	err = s.Put(locator, comment)
-	return comment, err
-}
-
-// Counts returns postID+count list for given comments
-func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
-	res := []store.PostInfo{}
-	for _, p := range postIDs {
-		if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
-			res = append(res, store.PostInfo{URL: p, Count: c})
-		}
-	}
-	return res, nil
-}
-
-// ValidateComment checks if comment size below max and user fields set
-func (s *DataStore) ValidateComment(c *store.Comment) error {
-	maxSize := s.MaxCommentSize
-	if s.MaxCommentSize <= 0 {
-		maxSize = defaultCommentMaxSize
-	}
-	if c.Orig == "" {
-		return errors.New("empty comment text")
-	}
-	if len([]rune(c.Orig)) > maxSize {
-		return errors.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, len([]rune(c.Orig)))
-	}
-	if c.User.ID == "" || c.User.Name == "" {
-		return errors.Errorf("empty user info")
-	}
-	return nil
-}
-
-// IsVerifiedFn returns func to check if user verified or not
-func (s *DataStore) IsVerifiedFn() func(siteID string, userID string) bool {
-	return func(siteID string, userID string) bool {
-		if siteID == "" {
-			return false
-		}
-		return s.IsVerified(siteID, userID)
-	}
-}
-
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
-	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
-
-	s.scopedLocks.Lock()
-	lock, ok := s.scopedLocks.locks[id]
-	if !ok {
-		lock = &sync.Mutex{}
-		s.scopedLocks.locks[id] = lock
-	}
-	s.scopedLocks.Unlock()
-
-	return lock
-}

app/store/service/service_test.go

@@ -1,372 +0,0 @@
-package service
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	id, err := b.Create(comment)
-	assert.NoError(t, err)
-	assert.True(t, id != "", id)
-
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
-	assert.NoError(t, err)
-	t.Logf("%+v", res)
-	assert.Equal(t, "text", res.Text)
-	assert.True(t, time.Since(res.Timestamp).Seconds() < 1)
-	assert.Equal(t, "user", res.User.ID)
-	assert.Equal(t, "name", res.User.Name)
-	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, map[string]bool{}, res.Votes)
-}
-
-func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
-	comment := store.Comment{
-		Text:      "text",
-		Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
-		Votes:     map[string]bool{"u1": true, "u2": false},
-		User:      store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	id, err := b.Create(comment)
-	assert.NoError(t, err)
-	assert.True(t, id != "", id)
-
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
-	assert.NoError(t, err)
-	t.Logf("%+v", res)
-	assert.Equal(t, "text", res.Text)
-	assert.Equal(t, comment.Timestamp, res.Timestamp)
-	assert.Equal(t, "user", res.User.ID)
-	assert.Equal(t, "name", res.User.Name)
-	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, comment.Votes, res.Votes)
-}
-
-func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
-
-	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, c.Score)
-	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
-
-	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
-	assert.NotNil(t, err, "self-voting not allowed")
-
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-	assert.NotNil(t, err, "double-voting rejected")
-	assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted"))
-
-	res, err = b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 1, res[0].Score)
-
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
-	assert.Nil(t, err, "vote reset")
-	res, err = b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "vote reset ok")
-}
-
-func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-
-	res, err := b.Last("radio-t", 0)
-	require.Nil(t, err)
-	t.Logf("%+v", res[0])
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
-
-	// add a vote as user2
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
-	require.Nil(t, err)
-
-	// crazy vote +1 as user1
-	var wg sync.WaitGroup
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-
-	t.Logf("%+v", res[0])
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 2, res[0].Score, "add single +1")
-	assert.Equal(t, 2, len(res[0].Votes), "made a single vote")
-
-	// random +1/-1 result should be [0..2]
-	rand.Seed(time.Now().UnixNano())
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-	assert.Equal(t, 3, len(res))
-	t.Logf("%+v %d", res[0], res[0].Score)
-	assert.True(t, res[0].Score >= 0 && res[0].Score <= 2, "unexpected score %d", res[0].Score)
-}
-
-func TestService_VoteConcurrent(t *testing.T) {
-
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-	res, err := b.Last("radio-t", 0)
-	require.Nil(t, err)
-
-	// concurrent vote +1 as multiple users for the same comment
-	var wg sync.WaitGroup
-	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-	assert.Equal(t, 100, res[0].Score, "should have 1000 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 1000 votes")
-}
-
-func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, false, res[0].Pin)
-
-	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true)
-	assert.Nil(t, err)
-
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, true, c.Pin)
-
-	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false)
-	assert.Nil(t, err)
-	c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, false, c.Pin)
-}
-
-func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Nil(t, res[0].Edit)
-
-	comment, err := b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err)
-	assert.Equal(t, "my edit", comment.Edit.Summary)
-	assert.Equal(t, "xxx", comment.Text)
-	assert.Equal(t, "yyy", comment.Orig)
-
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "my edit", c.Edit.Summary)
-	assert.Equal(t, "xxx", c.Text)
-
-	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err, "allow second edit")
-}
-
-func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Nil(t, res[0].Edit)
-
-	time.Sleep(time.Second)
-
-	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.NotNil(t, err)
-}
-
-func TestService_ValidateComment(t *testing.T) {
-
-	b := DataStore{MaxCommentSize: 2000}
-	longText := fmt.Sprintf("%4000s", "X")
-
-	tbl := []struct {
-		inp store.Comment
-		err error
-	}{
-		{inp: store.Comment{}, err: errors.New("empty comment text")},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: nil},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: errors.New("empty user info")},
-		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: errors.New("comment text exceeded max allowed size 2000 (4000)")},
-	}
-
-	for n, tt := range tbl {
-		e := b.ValidateComment(&tt.inp)
-		if tt.err == nil {
-			assert.Nil(t, e, "check #%d", n)
-			continue
-		}
-		assert.EqualError(t, tt.err, e.Error(), "check #%d", n)
-	}
-}
-
-func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prepStoreEngine(t) // two comments for https://radio-t.com
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "123456",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	svc := DataStore{Interface: b}
-	res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"})
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res)
-
-	res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"})
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{
-		{URL: "https://radio-t.com", Count: 2},
-		{URL: "https://radio-t.com/2", Count: 1},
-		{URL: "blah", Count: 0},
-	}, res)
-}
-
-func TestService_IsVerifiedFn(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	fn := b.IsVerifiedFn()
-	verified := fn("radio-t", "user1")
-	assert.False(t, verified)
-
-	err := b.Interface.SetVerified("radio-t", "user1", true)
-	assert.Nil(t, err)
-	verified = fn("radio-t", "user1")
-	assert.True(t, verified)
-}
-
-// makes new boltdb, put two records
-func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b := boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/store/user.go

@@ -1,60 +0,0 @@
-package store
-
-import (
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
-	"hash/crc64"
-	"io"
-	"log"
-	"regexp"
-)
-
-// User holds user-related info
-type User struct {
-	Name     string `json:"name"`
-	ID       string `json:"id"`
-	Picture  string `json:"picture"`
-	IP       string `json:"ip,omitempty"`
-	Admin    bool   `json:"admin"`
-	Blocked  bool   `json:"block,omitempty"`
-	Verified bool   `json:"verified,omitempty"`
-}
-
-var reValidSha = regexp.MustCompile("^[a-fA-F0-9]{40}$")
-
-// HashIP replace IP field with hashed hmac
-func (u *User) HashIP(secret string) {
-	u.IP = HashValue(u.IP, secret)
-}
-
-// HashValue makes hmac with secret
-func HashValue(val string, secret string) string {
-	if val == "" || reValidSha.MatchString(val) {
-		return val // already hashed or empty
-	}
-	key := []byte(secret)
-	h := hmac.New(sha1.New, key)
-	if _, err := io.WriteString(h, val); err != nil {
-		// fail back to crc64
-		log.Printf("[WARN] can't hash ip, %s", err)
-		return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA)))
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}
-
-// EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
-// we need hashing for parts of id, in some others hashing for non-User values.
-func EncodeID(id string) string {
-	if reValidSha.MatchString(id) {
-		return id // already hashed or empty
-	}
-	h := sha1.New()
-	if _, err := io.WriteString(h, id); err != nil {
-		// fail back to crc64
-		log.Printf("[WARN] can't hash id %s, %s", id, err)
-		return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA)))
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}

backend/.golangci.yml

@@ -0,0 +1,78 @@
+run:
+  timeout: 5m
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0.1
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+# TODO: feel free to remove these excludes and fix the code
+      - hugeParam
+      - rangeValCopy
+      - singleCaseSwitch
+      - ifElseChain
+
+linters:
+  enable:
+    - megacheck
+    - golint
+    - govet
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+    - stylecheck
+    - gochecknoinits
+    - scopelint
+    - gocritic
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
+  disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "should have a package comment, unless it's in another file for this package"
+      linters:
+        - golint
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false
+
+service:
+  golangci-lint-version: 1.31.x

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,22 @@
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+#ADD . /build/memory_store
+#WORKDIR /build/memory_store
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+
+FROM umputun/baseimage:app-latest
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,13 @@
+# sample store implementation 
+
+`memory_store` illustrates how to make a custom storage plugin for remark42. 
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. `docker-compose -f compose-dev-memstore.yml build`
+1. `docker-compose -f compose-dev-memstore.yml up`
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package.
+In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key(_ string) (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, errors.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", errors.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, errors.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return errors.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated++ // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,563 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.Lock()
+	defer m.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", errors.New("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.Lock()
+	defer m.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, errors.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.RLock()
+	defer m.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, errors.New("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.Lock()
+	defer m.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.RLock()
+	defer m.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		m.Lock()
+		defer m.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.Lock()
+			defer m.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.Lock()
+	defer m.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return errors.New("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return errors.New("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	return status, errors.Wrapf(err, "failed to set flag %+v", req)
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, errors.New("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID != comment.ID || c.Locator != comment.Locator {
+			continue
+		}
+		c.Text = comment.Text
+		c.Orig = comment.Orig
+		c.Score = comment.Score
+		c.Votes = comment.Votes
+		c.Pin = comment.Pin
+		c.Deleted = comment.Deleted
+		c.User = comment.User
+		comments[i] = c
+		m.posts[comment.Locator.SiteID] = comments
+		return nil
+	}
+	return errors.New("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,785 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds = make([]string, len(blockedList))
+	for i, x := range blockedList {
+		blockedIds[i] = x.ID
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		emailSet   = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.Unlock()
+
+	return nil
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.RUnlock()
+	if !ok {
+		return nil, errors.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.RLock()
+	img, ok := m.imagesStaging[id]
+	m.RUnlock()
+	if !ok {
+		return errors.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.Lock()
+	m.images[id] = img
+	m.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.RUnlock()
+
+	m.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := ioutil.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,65 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: '2'
+
+services:
+
+  remark42:
+    build:
+      context: ../../..
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42-dev"
+    hostname: "remark42-dev"
+    restart: always
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - BACKUP_PATH=/srv/var/backup
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: ../../..
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,14 @@
+module github.com/umputun/remark42/memory_store
+
+go 1.14
+
+require (
+	github.com/go-pkgz/jrpc v0.2.0
+	github.com/go-pkgz/lgr v0.10.4
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.6.1
+	github.com/umputun/go-flags v1.5.1
+	github.com/umputun/remark42/backend v1.6.0
+)
+
+replace github.com/umputun/remark42/backend => ../../

backend/_example/memory_store/go.sum

@@ -0,0 +1,296 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Depado/bfchroma v1.2.0 h1:NyYPFVhWvq8S2ts6Ok4kwXVE3TEO5fof+9ZOKbBJQUo=
+github.com/Depado/bfchroma v1.2.0/go.mod h1:U3RJUYwWVJrZRaJQyfS+wuxBApSTR/BC37PhAI+Ydps=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.6.0/go.mod h1:MmozekIi2rfQSzDcdEZ2BoJ9Pxs/7uc2Y4Boh+hIeZo=
+github.com/alecthomas/chroma v0.7.2 h1:B76NU/zbQYIUhUowbi4fmvREmDUJLsUzKWTZmQd3ABY=
+github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.11.4/go.mod h1:VL3UDEfAH59bSa7MuHMuFToxkqyHh69s/WUbYlOAuyg=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth/v6 v6.0.1 h1:QvLvRpB1G2bzKvkRze0muMUBlGN9H1z7tJ4DH4ypWOU=
+github.com/didip/tollbooth/v6 v6.0.1/go.mod h1:j2pKs+JQ5PvU/K4jFnrnwntrmfUbYLJE5oSdxR37FD0=
+github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9 h1:gTh8fKuI/yLqQtZEPlDX3ZGsiTPZIe0ADHsxXSbwO1I=
+github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
+github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.11.0/go.mod h1:NzVqlTW0E9JXVdAaWRq81XZjICgHnNaNdUfE3CbS2T4=
+github.com/go-pkgz/auth v1.14.0/go.mod h1:1GVd61pXZcuJ0ZnOUdCTY08V8SreO7MJtsvEd5/WEWA=
+github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
+github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
+github.com/go-pkgz/jrpc v0.2.0 h1:CLy/eZyekjraVrxZV18N2R1mYLMJ/nWrgdfyIOGPY/E=
+github.com/go-pkgz/jrpc v0.2.0/go.mod h1:wd8vtQ4CgtCnuqua6x2b1SKIgv0VSOh5Dn0uUITbiUE=
+github.com/go-pkgz/lcw v0.7.1/go.mod h1:3P6g9QrJsDePXEMe42ywO+tW08L17tBJGwIDdI7lZ6g=
+github.com/go-pkgz/lcw v0.8.1/go.mod h1:Xw0/ZfApATgbjVPYRZO4XHdWyxAjErDWDWJ7TLlw1Vc=
+github.com/go-pkgz/lgr v0.7.0 h1:S/AAPwt/RE9a5mNJskA7dGVp+Dq6SMIW6LYjG3ITxY8=
+github.com/go-pkgz/lgr v0.7.0/go.mod h1:yMgxU+GobMRJgIEbSzDKy/67W18S7qmGx/7BVL5AB8Q=
+github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
+github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
+github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
+github.com/go-pkgz/rest v1.5.0 h1:C8SxXcXza4GiUUAn/95iCkvoIrGbS30qpwK19iqlrWQ=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
+github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/oauth2.v3 v3.12.0/go.mod h1:XEYgKqWX095YiPT+Aw5y3tCn+7/FMnlTFKrupgSiJ3I=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/_example/memory_store/main.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/umputun/go-flags"
+
+	"github.com/umputun/remark42/memory_store/accessor"
+	"github.com/umputun/remark42/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
+
+	rpcServer := jrpc.Server{
+		API:        opts.API,
+		AuthUser:   opts.AuthUser,
+		AuthPasswd: opts.AuthPasswd,
+		Version:    revision,
+		AppName:    "remark42-memory",
+		Logger:     log.Default(),
+	}
+
+	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID:  "remark",
+		Enabled: true,
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	key, err := s.adm.Key(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,306 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := ioutil.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/server/rpc.go

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+	"github.com/umputun/remark42/backend/app/store/engine"
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// RPC handler wraps both engine and remote server and implements all handlers for data store and admin store
+// Note: this file can be used as-is in any custom jrpc plugin
+type RPC struct {
+	*jrpc.Server
+	eng engine.Interface
+	adm admin.Store
+	img image.Store
+}
+
+// NewRPC makes RPC instance and register handlers
+func NewRPC(e engine.Interface, a admin.Store, i image.Store, r *jrpc.Server) *RPC {
+	res := &RPC{eng: e, adm: a, img: i, Server: r}
+	res.addHandlers()
+	return res
+}
+
+func (s *RPC) addHandlers() {
+	// data store handlers
+	s.Group("store", jrpc.HandlersGroup{
+		"create":      s.createHndl,
+		"find":        s.findHndl,
+		"get":         s.getHndl,
+		"update":      s.updateHndl,
+		"count":       s.countHndl,
+		"info":        s.infoHndl,
+		"flag":        s.flagHndl,
+		"list_flags":  s.listFlagsHndl,
+		"user_detail": s.userDetailHndl,
+		"delete":      s.deleteHndl,
+		"close":       s.closeHndl,
+	})
+
+	// admin store handlers
+	s.Group("admin", jrpc.HandlersGroup{
+		"key":     s.admKeyHndl,
+		"admins":  s.admAdminsHndl,
+		"email":   s.admEmailHndl,
+		"enabled": s.admEnabledHndl,
+		"event":   s.admEventHndl,
+	})
+
+	// image store handlers
+	s.Group("image", jrpc.HandlersGroup{
+		"save_with_id": s.imgSaveWithIDHndl,
+		"load":         s.imgLoadHndl,
+		"commit":       s.imgCommitHndl,
+		"cleanup":      s.imgCleanupHndl,
+		"info":         s.imgInfoHndl,
+	})
+}

backend/_example/memory_store/server/rpc_test.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/memory_store/accessor"
+)
+
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
+func prepTestStore(t *testing.T) (port int, teardown func()) {
+	mg := accessor.NewMemData()
+	adm := accessor.NewMemAdminStore("secret")
+	img := accessor.NewMemImageStore()
+	s := NewRPC(mg, adm, img, &jrpc.Server{API: "/test", Logger: jrpc.NoOpLogger})
+
+	admRec := accessor.AdminRec{
+		SiteID:  "test-site",
+		IDs:     []string{"id1", "id2"},
+		Email:   "admin@example.com",
+		Enabled: true,
+	}
+	adm.Set("test-site", admRec)
+
+	admRecDisabled := admRec
+	admRecDisabled.Enabled = false
+	adm.Set("test-site-disabled", admRecDisabled)
+
+	port = chooseRandomUnusedPort()
+	go func() {
+		_ = s.Run(port)
+	}()
+
+	waitForHTTPServerStart(port)
+
+	return port, func() {
+		require.NoError(t, s.Shutdown())
+	}
+}

backend/app/cmd/avatar.go

@@ -0,0 +1,85 @@
+package cmd
+
+import (
+	"path"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/go-pkgz/auth/avatar"
+)
+
+// AvatarCommand set of flags and command for avatar migration
+// it converts all avatars from src.type to dst.type.
+// Note: it is possible to run migration for the same types (src = dst) in order to resize all avatars.
+type AvatarCommand struct {
+	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
+	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
+
+	migrator AvatarMigrator
+	CommonOpts
+}
+
+// AvatarMigrator defines interface for migration
+type AvatarMigrator interface {
+	Migrate(avatar.Store, avatar.Store) (int, error)
+}
+
+type avatarMigrator struct{}
+
+// Migrate from one avatar store to another. Can be used to convert between stores
+func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
+	return avatar.Migrate(dst, src)
+}
+
+// Execute runs  with AvatarCommand parameters, entry point for "avatar" command
+func (ac *AvatarCommand) Execute(_ []string) error {
+	log.Printf("[INFO] migrate avatars from %s to %s", ac.AvatarSrc.Type, ac.AvatarDst.Type)
+
+	src, err := ac.makeAvatarStore(ac.AvatarSrc)
+	if err != nil {
+		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarSrc.Type)
+	}
+
+	dst, err := ac.makeAvatarStore(ac.AvatarDst)
+	if err != nil {
+		return errors.Wrapf(err, "can't make avatart store for %s", ac.AvatarDst.Type)
+	}
+
+	if ac.migrator == nil {
+		ac.migrator = avatarMigrator{}
+	}
+
+	count, err := ac.migrator.Migrate(dst, src)
+	if err != nil {
+		return err
+	}
+
+	if err = dst.Close(); err != nil {
+		log.Printf("[WARN] failed to close dst store %s", ac.AvatarDst.Type)
+	}
+	if err = src.Close(); err != nil {
+		log.Printf("[WARN] failed to close src store %s", ac.AvatarSrc.Type)
+	}
+
+	log.Printf("[INFO] completed, migrated avatars = %d", count)
+	return nil
+}
+
+func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
+	log.Printf("[DEBUG] make avatar store, type=%s", gr.Type)
+	switch gr.Type {
+	case "fs":
+		if err := makeDirs(gr.FS.Path); err != nil {
+			return nil, errors.Wrap(err, "failed to create avatar store")
+		}
+		return avatar.NewLocalFS(gr.FS.Path), nil
+	case "bolt":
+		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
+			return nil, errors.Wrap(err, "failed to create avatar store")
+		}
+		return avatar.NewBoltDB(gr.Bolt.File, bolt.Options{})
+	}
+	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
+}

backend/app/cmd/avatar_test.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/go-pkgz/auth/avatar"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
+)
+
+func TestAvatar_Execute(t *testing.T) {
+
+	defer os.RemoveAll("/tmp/ava-test")
+
+	// from fs to bolt
+	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
+	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test.db"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+
+	// failed
+	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
+	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
+	p = flags.NewParser(&cmd, flags.Default)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test2.db"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.Error(t, err, "failed blah")
+}
+
+type avatarMigratorMock struct {
+	called   int
+	retError error
+	retCount int
+}
+
+func (a *avatarMigratorMock) Migrate(_, _ avatar.Store) (int, error) {
+	a.called++
+	return a.retCount, a.retError
+}

backend/app/cmd/backup.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// BackupCommand set of flags and command for export
+// ExportPath used as a separate element to leverage BACKUP_PATH. If ExportFile has a path (i.e. with /) BACKUP_PATH ignored.
+type BackupCommand struct {
+	ExportPath  string        `short:"p" long:"path" env:"BACKUP_PATH" default:"./var/backup" description:"export path"`
+	ExportFile  string        `short:"f" long:"file" default:"userbackup-{{.SITE}}-{{.TS}}.gz" description:"file name"`
+	Site        string        `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Timeout     time.Duration `long:"timeout" default:"15m" description:"export (backup) timeout"`
+	AdminPasswd string        `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	CommonOpts
+}
+
+// Execute runs export with ExportCommand parameters, entry point for "export" command
+func (ec *BackupCommand) Execute(_ []string) error {
+	log.Printf("[INFO] export to %s, site %s", ec.ExportPath, ec.Site)
+	resetEnv("SECRET", "ADMIN_PASSWD")
+
+	fp := fileParser{site: ec.Site, path: ec.ExportPath, file: ec.ExportFile}
+	fname, err := fp.parse(time.Now())
+	if err != nil {
+		return err
+	}
+
+	log.Printf("[DEBUG] export file %s", fname)
+
+	// prepare http client and request
+	client := http.Client{}
+	ctx, cancel := context.WithTimeout(context.Background(), ec.Timeout)
+	defer cancel()
+	exportURL := fmt.Sprintf("%s/api/v1/admin/export?mode=file&site=%s", ec.RemarkURL, ec.Site)
+	req, err := http.NewRequest(http.MethodGet, exportURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "can't make export request for %s", exportURL)
+	}
+	req.SetBasicAuth("admin", ec.AdminPasswd)
+
+	// get with timeout
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.Wrapf(err, "request failed for %s", exportURL)
+	}
+	defer func() {
+		if err = resp.Body.Close(); err != nil {
+			log.Printf("[WARN] failed to close response, %s", err)
+		}
+	}()
+
+	if resp.StatusCode >= 300 {
+		return responseError(resp)
+	}
+
+	fh, err := os.Create(fname)
+	if err != nil {
+		return errors.Wrapf(err, "can't create backup file %s", fname)
+	}
+	defer func() {
+		if err = fh.Close(); err != nil {
+			log.Printf("[WARN] failed to close file %s, %s", fh.Name(), err)
+		}
+	}()
+
+	if _, err = io.Copy(fh, resp.Body); err != nil {
+		return errors.Wrapf(err, "failed to write backup file %s", fname)
+	}
+
+	log.Printf("[INFO] export completed, file %s", fname)
+	return nil
+}

backend/app/cmd/backup_test.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/umputun/go-flags"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBackup_Execute(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		fmt.Fprint(w, "blah\nblah2\n12345678\n")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	defer os.Remove("/tmp/remark-test.export")
+
+	data, err := ioutil.ReadFile("/tmp/remark-test.export")
+	require.NoError(t, err)
+	assert.Equal(t, "blah\nblah2\n12345678\n", string(data))
+}
+
+func TestBackup_ExecuteFailedStatus(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		w.WriteHeader(400)
+		fmt.Fprint(w, "some error")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp", "--file={{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, `error response "400 Bad Request", some error`)
+}
+
+func TestBackup_ExecuteFailedWrite(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.URL.Path, "/api/v1/admin/export")
+		assert.Equal(t, "GET", r.Method)
+		fmt.Fprint(w, "blah\nblah2\n12345678\n")
+	}))
+	defer ts.Close()
+
+	cmd := BackupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--path=/tmp",
+		"--file=/tmp/no-such-dir/{{.SITE}}-test.export", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.EqualError(t, err, `can't create backup file /tmp/no-such-dir/remark-test.export: open /tmp/no-such-dir/remark-test.export: no such file or directory`)
+}

backend/app/cmd/cleanup.go

@@ -0,0 +1,292 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store"
+)
+
+// CleanupCommand set of flags and command for cleanup
+type CleanupCommand struct {
+	Site        string   `short:"s" long:"site" env:"SITE" default:"remark" description:"site name"`
+	Dry         bool     `long:"dry" description:"dry mode, will not remove comments"`
+	From        string   `long:"from" description:"from yyyymmdd"`
+	To          string   `long:"to" description:"from yyyymmdd"`
+	BadWords    []string `short:"w" long:"bword" description:"bad word(s)"`
+	BadUsers    []string `short:"u" long:"buser" description:"bad user(s)"`
+	AdminPasswd string   `long:"admin-passwd" env:"ADMIN_PASSWD" required:"true" description:"admin basic auth password"`
+	SetTitle    bool     `long:"title" description:"title mode, will not remove comments, but reset titles to page's title'"`
+	CommonOpts
+}
+
+var (
+	defaultFrom = time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local)
+	defaultTo   = time.Date(2999, 1, 1, 0, 0, 0, 0, time.Local)
+)
+
+// Execute runs cleanup with CleanupCommand parameters, entry point for "cleanup" command
+// This command uses provided flags to detect and remove junk comments
+func (cc *CleanupCommand) Execute(_ []string) error {
+	log.Printf("[INFO] cleanup for site %s", cc.Site)
+
+	posts, err := cc.postsInRange(cc.From, cc.To)
+	if err != nil {
+		return errors.Wrap(err, "can't get posts")
+	}
+	log.Printf("[DEBUG] got %d posts", len(posts))
+
+	totalComments, spamComments := 0, 0
+	for _, post := range posts {
+		comments, e := cc.listComments(post.URL)
+		if e != nil {
+			continue
+		}
+		totalComments += len(comments)
+
+		if cc.SetTitle {
+			cc.procTitles(comments)
+		} else {
+			spamComments += cc.procSpam(comments)
+
+		}
+	}
+
+	msg := fmt.Sprintf("comments=%d, spam=%d", totalComments, spamComments)
+	if cc.SetTitle {
+		msg = fmt.Sprintf("comments=%d", totalComments)
+	}
+
+	log.Printf("[INFO] completed, %s", msg)
+	return err
+}
+
+func (cc *CleanupCommand) procSpam(comments []store.Comment) int {
+	spamComments := 0
+	for _, comment := range comments {
+		spam, score := cc.isSpam(comment)
+		if spam {
+			spamComments++
+			if !cc.Dry {
+				if err := cc.deleteComment(comment); err != nil {
+					log.Printf("[WARN] can't remove comment, %v", err)
+				}
+			}
+			comment.Text = strings.Replace(comment.Text, "\n", " ", -1)
+			log.Printf("[SPAM] %+v [%.0f%%]", comment, score)
+		}
+	}
+	return spamComments
+}
+
+func (cc *CleanupCommand) procTitles(comments []store.Comment) {
+	for _, comment := range comments {
+		if !cc.Dry {
+			if err := cc.setTitle(comment); err != nil {
+				log.Printf("[WARN] can't set title for comment, %v", err)
+			}
+		}
+	}
+}
+
+// get list of posts in from/to represented as yyyymmdd. this is [from-to] inclusive
+func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, error) {
+	posts, err := cc.listPosts()
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't list posts for %s", cc.Site)
+	}
+
+	from, to := defaultFrom, defaultTo
+
+	if fromS != "" {
+		from, err = time.ParseInLocation("20060102", fromS, time.Local)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't parse --from")
+		}
+	}
+
+	if toS != "" {
+		to, err = time.ParseInLocation("20060102", toS, time.Local)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't parse --to")
+		}
+	}
+
+	var filteredList []store.PostInfo
+	for _, postInfo := range posts {
+		if postInfo.FirstTS.After(from) && postInfo.LastTS.Before(to.AddDate(0, 0, 1)) {
+			filteredList = append(filteredList, postInfo)
+		}
+	}
+	return filteredList, nil
+}
+
+// get all posts via GET /list?site=siteID&limit=50&skip=10
+func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
+	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
+	client := http.Client{Timeout: 30 * time.Second}
+	r, err := client.Get(listURL)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
+	}
+	defer func() { _ = r.Body.Close() }()
+
+	if r.StatusCode != 200 {
+		return nil, errors.Errorf("request %s failed with status %d", listURL, r.StatusCode)
+	}
+
+	list := []store.PostInfo{}
+	if err = json.NewDecoder(r.Body).Decode(&list); err != nil {
+		return nil, errors.Wrapf(err, "can't decode list of posts for site %s", cc.Site)
+	}
+	return list, nil
+}
+
+// get all comments for post url via /find?site=siteID&url=post-url&format=[tree|plain]
+func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error) {
+
+	commentsURL := fmt.Sprintf("%s/api/v1/find?site=%s&url=%s&format=plain", cc.RemarkURL, cc.Site, postURL)
+
+	var r *http.Response
+	var err error
+
+	// handle 429 error from limiter
+	for {
+		client := http.Client{Timeout: 30 * time.Second}
+		r, err = client.Get(commentsURL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			_ = r.Body.Close()
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		break
+	}
+
+	defer func() { _ = r.Body.Close() }()
+
+	if r.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("request %s failed with status %d", commentsURL, r.StatusCode)
+	}
+
+	commentsWithInfo := struct {
+		Comments []store.Comment `json:"comments"`
+		Info     store.PostInfo  `json:"info,omitempty"`
+	}{}
+
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
+	}
+	return commentsWithInfo.Comments, nil
+}
+
+// deleteComment with DELETE /admin/comment/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) deleteComment(c store.Comment) error {
+
+	deleteURL := fmt.Sprintf("%s/api/v1/admin/comment/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+	req, err := http.NewRequest("DELETE", deleteURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "failed to make delete request for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	req.SetBasicAuth("admin", cc.AdminPasswd)
+
+	client := http.Client{}
+	r, err := client.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "delete request failed for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	defer func() { _ = r.Body.Close() }()
+	if r.StatusCode != http.StatusOK {
+		return errors.Errorf("delete request failed with status %s", r.Status)
+	}
+	return nil
+}
+
+// setTitle with PUT /admin/title/{id}?site=siteID&url=post-url
+func (cc *CleanupCommand) setTitle(c store.Comment) error {
+
+	titleURL := fmt.Sprintf("%s/api/v1/admin/title/%s?site=%s&url=%s&format=plain", cc.RemarkURL, c.ID, cc.Site, c.Locator.URL)
+	req, err := http.NewRequest("PUT", titleURL, nil)
+	if err != nil {
+		return errors.Wrapf(err, "failed to make title request for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	req.SetBasicAuth("admin", cc.AdminPasswd)
+
+	client := http.Client{}
+	r, err := client.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "title request failed for comment %s, %s", c.ID, c.Locator.URL)
+	}
+	defer func() { _ = r.Body.Close() }()
+	if r.StatusCode != http.StatusOK {
+		return errors.Errorf("title request failed with status %s", r.Status)
+	}
+	return nil
+}
+
+// isSpam calculates spam's probability as a score
+func (cc *CleanupCommand) isSpam(comment store.Comment) (isSpam bool, spamScore float64) {
+
+	badWord := func(txt string) float64 {
+		res := 0.0
+		for _, w := range cc.BadWords {
+			if strings.Contains(txt, w) {
+				res += 0.25
+			}
+			if res > 1 {
+				return 1
+			}
+		}
+		return res
+	}
+
+	hasBadUser := func(txt string) bool {
+		for _, w := range cc.BadUsers {
+			if strings.Contains(txt, w) {
+				return true
+			}
+		}
+		return false
+	}
+
+	score := 0.0
+
+	// don't mark deleted as spam
+	if comment.Deleted {
+		return false, 0
+	}
+
+	score += 50 * badWord(comment.Text) // up to 50, 4 bad words will reach max
+
+	if hasBadUser(comment.User.ID) { // predefined list of bad user substrings
+		score += 10
+	}
+
+	if comment.Score == 0 { // most of spam comments with 0 score
+		score += 20
+	}
+
+	// any link inside
+	if strings.Contains(comment.Text, "http:") || strings.Contains(comment.Text, "https:") {
+		score += 10
+	}
+
+	// 5 or more links
+	if strings.Count(comment.Text, "href") >= 5 {
+		score += 10
+	}
+
+	score = math.Max(score, 0)
+	score = math.Min(score, 100)
+
+	return score > 50, score
+}

backend/app/cmd/cleanup_test.go

@@ -0,0 +1,215 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/umputun/go-flags"
+
+	"github.com/umputun/remark42/backend/app/store"
+)
+
+type cleanedComments struct {
+	ids  []string
+	lock sync.Mutex
+}
+
+func TestCleanup_IsSpam(t *testing.T) {
+	cc := CleanupCommand{
+		BadWords: []string{"bad1", "bad2", "very bad", "xyz"},
+		BadUsers: []string{"bu_"},
+	}
+
+	tbl := []struct {
+		text      string
+		user      string
+		score     int
+		isSpam    bool
+		spamScore float64
+		name      string
+	}{
+		{"", "", 1, false, 0, "empty passes"},
+		{"one very bad two blah bad1 bad2 http://xyz.com", "bu_user", 0, true, 90, "3badwords link 0score baduser"},
+		{"one very bad two blah bad1 bad2", "bu_user", 0, true, 67.5, "3 bad words 1score baduser"},
+		{"bad1 bad2 xyz very bad", "bu_user", 0, true, 80, "4badwords 0score baduser"},
+		{"bad1 bad2 xyz very bad", "user", 0, true, 70, "4badwords 0score"},
+		{"bad1 bad2 xyz very bad", "user", 1, false, 50, "4badwords 1score"},
+		{"bad1 test 12345", "user", 0, false, 32.5, "1badwords 0score"},
+	}
+
+	for n, tt := range tbl {
+		tt := tt
+		checkName := fmt.Sprintf("check-%d-%s", n, tt.name)
+		t.Run(checkName, func(t *testing.T) {
+			c := store.Comment{ID: checkName, Text: tt.text, Score: tt.score}
+			c.User.ID = tt.user
+			r, score := cc.isSpam(c)
+			assert.Equal(t, tt.isSpam, r)
+			assert.InDelta(t, tt.spamScore, score, 0.01)
+		})
+	}
+}
+
+func TestCleanup_postsInRange(t *testing.T) {
+
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	posts, err := cmd.postsInRange("20181218", "20181219")
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(posts))
+
+	posts, err = cmd.postsInRange("", "")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(posts))
+
+	_, err = cmd.postsInRange("xxx", "yyy")
+	assert.Error(t, err)
+}
+
+func TestCleanup_listComments(t *testing.T) {
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, nil)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_", "--admin-passwd=secret"})
+	require.NoError(t, err)
+
+	comments, err := cmd.listComments("http://test.com/post1")
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post2")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+
+	comments, err = cmd.listComments("http://test.com/post-bad")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+}
+
+func TestCleanup_ExecuteSpam(t *testing.T) {
+	cleaned := cleanedComments{}
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, &cleaned)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--bword=bad1", "--bword=bad2", "--buser=bu_",
+		"--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("deleted %+v", cleaned.ids)
+	assert.Equal(t, []string{"/api/v1/admin/comment/1", "/api/v1/admin/comment/3", "/api/v1/admin/comment/11"}, cleaned.ids)
+}
+
+func TestCleanup_ExecuteTitle(t *testing.T) {
+	titledComments := cleanedComments{}
+	r := chi.NewRouter()
+	cleanupRoutes(t, r, &titledComments)
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	cmd := CleanupCommand{}
+	cmd.SetCommon(CommonOpts{RemarkURL: ts.URL, SharedSecret: "123456"})
+	p := flags.NewParser(&cmd, flags.Default)
+	_, err := p.ParseArgs([]string{"--site=remark", "--title", "--from=20181217", "--to=20181218", "--admin-passwd=secret"})
+	require.NoError(t, err)
+	err = cmd.Execute(nil)
+	assert.NoError(t, err)
+	t.Logf("set titles for %+v", titledComments.ids)
+	assert.Equal(t, []string{"/api/v1/admin/title/1", "/api/v1/admin/title/2", "/api/v1/admin/title/3", "/api/v1/admin/title/11"}, titledComments.ids)
+}
+
+func cleanupRoutes(t *testing.T, r *chi.Mux, c *cleanedComments) {
+	r.HandleFunc("/api/v1/list", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "site=remark&limit=10000", r.URL.RawQuery)
+		list := []store.PostInfo{
+			{
+				URL:     "http://test.com/post1",
+				FirstTS: time.Date(2018, 12, 17, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 17, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post2",
+				FirstTS: time.Date(2018, 12, 18, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 18, 10, 30, 0, 0, time.Local),
+			},
+			{
+				URL:     "http://test.com/post3",
+				FirstTS: time.Date(2018, 12, 19, 10, 0, 0, 0, time.Local),
+				LastTS:  time.Date(2018, 12, 19, 10, 30, 0, 0, time.Local),
+			},
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(list))
+	})
+
+	r.HandleFunc("/api/v1/find", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		require.Equal(t, "remark", r.URL.Query().Get("site"))
+		require.Equal(t, "plain", r.URL.Query().Get("format"))
+
+		commentsWithInfo := struct {
+			Comments []store.Comment `json:"comments"`
+			Info     store.PostInfo  `json:"info,omitempty"`
+		}{}
+
+		switch r.URL.Query().Get("url") {
+		case "http://test.com/post1":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "1", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+				{ID: "2", Text: "good one http://xyz.com", Score: 1, User: store.User{ID: "bu_user"}},
+				{ID: "3", Text: "http://xyz.com bad1 bad2", Score: 0, User: store.User{ID: "user"}},
+			}
+		case "http://test.com/post2":
+			commentsWithInfo.Comments = []store.Comment{
+				{ID: "11", Text: "one very bad two blah bad1 bad2 http://xyz.com", Score: 0, User: store.User{ID: "bu_user"}},
+			}
+		case "http://test.com/post3":
+			commentsWithInfo.Comments = []store.Comment{}
+		}
+
+		require.NoError(t, json.NewEncoder(w).Encode(commentsWithInfo))
+	})
+
+	r.HandleFunc("/api/v1/admin/comment/{id}", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "DELETE", r.Method)
+		t.Log("delete ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	})
+
+	r.HandleFunc("/api/v1/admin/title/{id}", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PUT", r.Method)
+		t.Log("title for ", r.URL.Path)
+		c.lock.Lock()
+		c.ids = append(c.ids, r.URL.Path)
+		c.lock.Unlock()
+	})
+
+}

backend/app/cmd/cmd.go

@@ -0,0 +1,121 @@
+// Package cmd has all top-level commands dispatched by main's flag.Parse
+// The entry point of each command is Execute function
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+)
+
+// CommonOptionsCommander extends flags.Commander with SetCommon
+// All commands should implement this interfaces
+type CommonOptionsCommander interface {
+	SetCommon(commonOpts CommonOpts)
+	Execute(args []string) error
+	HandleDeprecatedFlags() []DeprecatedFlag
+}
+
+// CommonOpts sets externally from main, shared across all commands
+type CommonOpts struct {
+	RemarkURL    string
+	SharedSecret string
+	Revision     string
+}
+
+// DeprecatedFlag contains information about deprecated option
+type DeprecatedFlag struct {
+	Old           string
+	New           string
+	RemoveVersion string
+}
+
+// SetCommon satisfies CommonOptionsCommander interface and sets common option fields
+// The method called by main for each command
+func (c *CommonOpts) SetCommon(commonOpts CommonOpts) {
+	c.RemarkURL = strings.TrimSuffix(commonOpts.RemarkURL, "/") // allow RemarkURL with trailing /
+	c.SharedSecret = commonOpts.SharedSecret
+	c.Revision = commonOpts.Revision
+}
+
+// HandleDeprecatedFlags sets new flags from deprecated and returns their list
+func (c *CommonOpts) HandleDeprecatedFlags() []DeprecatedFlag { return nil }
+
+// fileParser used to convert template strings like blah-{{.SITE}}-{{.YYYYMMDD}} the final format
+type fileParser struct {
+	site string
+	file string
+	path string
+}
+
+// parse apply template and also concat path and file. In case if file contains path separator path will be ignored
+func (p *fileParser) parse(now time.Time) (string, error) {
+
+	// file/location parameters my have template masks
+	fileTemplate := struct {
+		YYYYMMDD string
+		YYYY     string
+		YYYYMM   string
+		MM       string
+		DD       string
+		TS       string
+		UNIX     int64
+		SITE     string
+	}{
+		YYYYMMDD: now.Format("20060102"),
+		YYYY:     now.Format("2006"),
+		YYYYMM:   now.Format("200601"),
+		MM:       now.Format("01"),
+		DD:       now.Format("02"),
+		UNIX:     now.Unix(),
+		SITE:     p.site,
+		TS:       now.Format("20060102T150405"),
+	}
+
+	bb := bytes.Buffer{}
+	fname := p.file
+	if !strings.Contains(p.file, string(filepath.Separator)) {
+		fname = filepath.Join(p.path, p.file)
+	}
+
+	if err := template.Must(template.New("bb").Parse(fname)).Execute(&bb, fileTemplate); err != nil {
+		return "", errors.Wrapf(err, "failed to parse %q", fname)
+	}
+	return bb.String(), nil
+}
+
+// resetEnv clears sensitive env vars
+func resetEnv(envs ...string) {
+	for _, env := range envs {
+		if err := os.Unsetenv(env); err != nil {
+			log.Printf("[WARN] can't unset env %s, %s", env, err)
+		}
+	}
+}
+
+// responseError returns error with status and response body
+func responseError(resp *http.Response) error {
+	body, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		body = []byte("")
+	}
+	return errors.Errorf("error response %q, %s", resp.Status, body)
+}
+
+// mkdir -p for all dirs
+func makeDirs(dirs ...string) error {
+	for _, dir := range dirs {
+		if err := os.MkdirAll(dir, 0700); err != nil { // If path is already a directory, MkdirAll does nothing
+			return errors.Wrapf(err, "can't make directory %s", dir)
+		}
+	}
+	return nil
+}