attempt to fix ci

another fix for ci

fix build tagged images

.dockerignore

@@ -1,11 +1,11 @@
 /logs/
 /target/
 /var/
-/web/node_modules/
-/web/public/
+/frontend/node_modules/
+/frontend/public/
 /.vscode/
 /.idea/
-#/.git/objects/
+/bin/
 
 # source files
 docker-compose.yml
@@ -19,3 +19,6 @@ debug
 debug.test
 *.prof
 *.test
+remark42
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -1,80 +0,0 @@
-workspace:
-  base: /go
-  path: src/github.com/umputun/remark
-
-pipeline:
-  
-  build:
-    image: golang:1.10-alpine
-    commands:
-      - cd app
-      - go build -v ./...
-
-  docker_master:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    tags: 
-      - ${DRONE_COMMIT_BRANCH/\//-}
-    when:
-      branch: [master, release/*]
-      event: push
-
-  docker_tag:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    build_args:
-      - DRONE_TAG=${DRONE_TAG}
-      - DRONE_COMMIT=${DRONE_COMMIT}
-    tags: 
-      - ${DRONE_TAG}
-      - latest
-    when:
-      event: tag
-
-  docker_branch:
-    image: plugins/docker
-    repo: umputun/remark42
-    secrets: [ docker_username, docker_password ]
-    tags: 
-      - ${DRONE_COMMIT_BRANCH/\//-}
-    when:
-      branch:
-        exclude: [master, release/*]
-        event: push
-
-  docker_pullrequest:
-    image: docker
-    commands:
-      - docker build .
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    when:
-      event: pull_request
-
-  deploy:
-    image: appleboy/drone-ssh
-    host: remark42.com
-    username: umputun
-    port: 22
-    secrets: [ ssh_key ]
-    script:
-      - cd /srv/remark
-      - docker-compose pull
-      - docker-compose up -d
-    when:
-      branch: master
-      event: push
-
-  notify:
-    image: drillster/drone-email
-    host: smtp.mailgun.org
-    port: 25
-    username: email_username
-    password: email_password
-    from: drone@mg.umputun.com
-    recipients: [ umputun@gmail.com ]
-    secrets: [ email_username, email_password ]
-    when:
-      status: [ changed, failure ]

.editorconfig

@@ -0,0 +1,11 @@
+[*]
+indent_style = tab
+insert_final_newline = true
+
+[*.md]
+indent_style = space
+trim_trailing_whitespace = false
+
+[*.yml]
+indent_size = 2
+indent_style = space

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence, @umputun will be requested for
+# review when someone opens a pull request.
+
+*       @umputun

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [umputun]
+patreon: remark42

.github/workflows/ci-build.yml

@@ -0,0 +1,85 @@
+name: build
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - ".github/workflows/ci-build.yml"
+      - "backend/**"
+      - "frontend/**"
+      - ".dockerignore"
+      - "docker-init.sh"
+      - "Dockerfile"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-build.yml"
+      - "backend/**"
+      - "frontend/**"
+      - ".dockerignore"
+      - "docker-init.sh"
+      - "Dockerfile"
+      - "!**.md"
+
+jobs:
+  build-images:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: build and deploy master image to ghcr.io and dockerhub
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo GITHUB_REF - $ref
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u ${USERNAME} --password-stdin
+          docker buildx build --push \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
+              --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} \
+              --platform linux/amd64,linux/arm/v7,linux/arm64 \
+              -t ghcr.io/${USERNAME}/remark42:${ref} -t ${USERNAME}/remark42:${ref} .
+
+      - name: deploy tagged (latest) to ghcr.io and dockerhub
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo "GITHUB_REF=$ref, GITHUB_SHA=${GITHUB_SHA}"
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          echo ${DOCKER_HUB_TOKEN} | docker login -u ${USERNAME} --password-stdin
+          docker buildx build --push \
+              --build-arg SKIP_BACKEND_TEST=true --build-arg SKIP_FRONTEND_TEST=true --build-arg CI=github \
+              --build-arg GITHUB_SHA=${GITHUB_SHA} --build-arg GIT_BRANCH=${ref} \
+              --platform linux/amd64,linux/arm/v7,linux/arm64 \
+              -t ghcr.io/umputun/remark42:${ref} -t ghcr.io/umputun/remark42:latest \
+              -t umputun/remark42:${ref} -t umputun/remark42:latest .
+
+      - name: remote deployment to remark42.com from master
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
+        run: curl https://jess.umputun.com/update/remark42-core/${UPDATER_KEY}

.github/workflows/ci-frontend-size-limit.yml

@@ -0,0 +1,20 @@
+name: frontend
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/ci-frontend-size-limit.yml"
+      - "frontend/**"
+      - "!**.md"
+
+jobs:
+  size:
+    runs-on: ubuntu-latest
+    env:
+      CI_JOB_NUMBER: 1
+    steps:
+      - uses: actions/checkout@v1
+      - uses: andresz1/size-limit-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          directory: frontend

.github/workflows/ci-frontend.yml

@@ -0,0 +1,107 @@
+name: frontend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - ".github/workflows/ci-frontend.yml"
+      - "frontend/**"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-frontend.yml"
+      - "frontend/**"
+      - "!**.md"
+
+jobs:
+  check-translations:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:translation
+        working-directory: ./frontend
+
+  check-typescript:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/frontend/node_modules/.cache
+          key: ${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - run: npm run check:types
+        working-directory: ./frontend
+
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npx run-p lint
+        working-directory: ./frontend
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [14.15]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm ci --loglevel warn
+        working-directory: ./frontend
+
+      - run: npm run test:coverage
+        working-directory: ./frontend
+
+      - name: submit coverage
+        run: node ${{ github.workspace }}/frontend/node_modules/.bin/codecov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/ci-site.yml

@@ -0,0 +1,68 @@
+name: site
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - ".github/workflows/ci-site.yml"
+      - "site/**"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-site.yml"
+      - "site/**"
+
+jobs:
+
+  build-site:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: build and deploy master image to ghcr.io and dockerhub
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        working-directory: ./site
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo GITHUB_REF - $ref
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
+            -t ghcr.io/${USERNAME}/remark24-site:${ref} .
+
+      - name: deploy tagged (latest) to ghcr.io and dockerhub
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GITHUB_PACKAGE_TOKEN: ${{ secrets.PKG_TOKEN }}
+          USERNAME: ${{ github.actor }}
+          GITHUB_SHA: ${{ github.sha}}
+          GITHUB_REF: ${{ github.ref}}
+        working-directory: ./site
+        run: |
+          ref="$(echo ${GITHUB_REF} | cut -d'/' -f3)"
+          echo "GITHUB_REF=$ref, GITHUB_SHA=${GITHUB_SHA}"
+          echo ${GITHUB_PACKAGE_TOKEN} | docker login ghcr.io -u ${USERNAME} --password-stdin
+          docker buildx build --push --no-cache --platform linux/amd64,linux/arm/v7,linux/arm64 \
+            -t ghcr.io/${USERNAME}/remark24-site:${ref} -t ghcr.io/${USERNAME}/remark24-site:latest .
+
+      - name: remote site deployment from master
+        if: ${{ github.ref == 'refs/heads/master' }}
+        env:
+          UPDATER_KEY: ${{ secrets.UPDATER_KEY }}
+        run: curl https://jess.umputun.com/update/remark42-site/${UPDATER_KEY}

.github/workflows/ci-test-backend.yml

@@ -0,0 +1,64 @@
+name: test_backend
+
+on:
+  push:
+    branches:
+    tags:
+    paths:
+      - ".github/workflows/ci-test-backend.yml"
+      - "backend/**"
+      - "!backend/scripts/**"
+      - "!**.md"
+  pull_request:
+    paths:
+      - ".github/workflows/ci-test-backend.yml"
+      - "backend/**"
+      - "!backend/scripts/**"
+      - "!**.md"
+jobs:
+  test-backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: debug if needed
+        run: if [[ "$DEBUG" == "true" ]]; then env; fi
+        env:
+          DEBUG: ${{secrets.DEBUG}}
+
+      - name: install go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - name: install golangci-lint and goveralls
+        run: |
+          curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $GITHUB_WORKSPACE v1.43.0
+          go get -u github.com/mattn/goveralls
+
+      - name: test and lint backend
+        run: |
+          go test -race -timeout=60s -covermode=atomic -coverprofile=$GITHUB_WORKSPACE/profile.cov_tmp ./...
+          cat $GITHUB_WORKSPACE/profile.cov_tmp | grep -v "_mock.go" > $GITHUB_WORKSPACE/profile.cov
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/app
+        env:
+          GOFLAGS: "-mod=vendor"
+          TZ: "America/Chicago"
+
+      - name: test and lint examples
+        run: |
+          go version
+          $GITHUB_WORKSPACE/golangci-lint version
+          go test -race ./...
+          $GITHUB_WORKSPACE/golangci-lint --config ${GITHUB_WORKSPACE}/backend/.golangci.yml run --out-format=github-actions ./...
+        working-directory: backend/_example/memory_store
+        env:
+          TZ: "America/Chicago"
+
+      - name: submit coverage
+        run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov
+        working-directory: backend
+        env:
+          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -8,9 +8,17 @@ debug
 debug.test
 .vscode
 .idea/
-/web/node_modules/
-/web/public/
 *.prof
 *.test
 /rest-client.env.json
 .DS_Store
+.mongo
+remark42
+/bin/
+/backend/var/
+/backend/app/var/
+compose-private-backend.yml
+compose-private-frontend.yml
+compose-private.yml
+/backend/_example/*/vendor
+http-client.env.json

.travis.yml

@@ -1,19 +0,0 @@
-install:
-  - docker --version
-  - docker-compose --version
-
-script:
-  - docker build
-      --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
-      --build-arg CI=$CI
-      --build-arg TRAVIS=$TRAVIS
-      --build-arg TRAVIS_BRANCH=$TRAVIS_BRANCH
-      --build-arg TRAVIS_COMMIT=$TRAVIS_COMMIT
-      --build-arg TRAVIS_JOB_ID=$TRAVIS_JOB_ID
-      --build-arg TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER
-      --build-arg TRAVIS_OS_NAME=$TRAVIS_OS_NAME
-      --build-arg TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-      --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
-      --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
-      --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      .

Dockerfile

@@ -1,77 +1,82 @@
-FROM umputun/baseimage:buildgo-latest as build-backend
+FROM umputun/baseimage:buildgo-v1.8.0 as build-backend
 
-ARG COVERALLS_TOKEN
 ARG CI
-ARG TRAVIS
-ARG TRAVIS_BRANCH
-ARG TRAVIS_COMMIT
-ARG TRAVIS_JOB_ID
-ARG TRAVIS_JOB_NUMBER
-ARG TRAVIS_OS_NAME
-ARG TRAVIS_PULL_REQUEST
-ARG TRAVIS_PULL_REQUEST_SHA
-ARG TRAVIS_REPO_SLUG
-ARG TRAVIS_TAG
+ARG DRONE
 ARG DRONE_TAG
 ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+ARG DRONE_PULL_REQUEST
 
-WORKDIR /go/src/github.com/umputun/remark
+ARG SKIP_BACKEND_TEST
+ARG BACKEND_TEST_TIMEOUT
 
-ADD app /go/src/github.com/umputun/remark/app
-ADD vendor /go/src/github.com/umputun/remark/vendor
-ADD .git /go/src/github.com/umputun/remark/.git
+ADD backend /build/backend
+ADD .git/ /build/backend/.git/
+WORKDIR /build/backend
 
-RUN cd app && go test ./...
+ENV GOFLAGS="-mod=vendor"
 
-RUN gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
-    --enable=deadcode  --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./...
-
-RUN mkdir -p target && /script/coverage.sh
-
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
-    echo coverall not enabled ; \
-    else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi
+# install gcc in order to be able to go test package with -race
+RUN apk --no-cache add gcc libc-dev
 
+# run tests
 RUN \
-    version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --abbrev=7 --always --tags)-$(date +%Y%m%d-%H:%M:%S) && \
-    echo "git version=$version" && \  
-    if [ -z "$DRONE_TAG" ] ; then \
-    echo "runs outside of drone" ; \
-    else version=${DRONE_TAG}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
-    echo "final version=$version" && \  
-    go build -o remark -ldflags "-X main.revision=${version} -s -w" ./app
+    cd app && \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        CGO_ENABLED=1 go test -race -p 1 -timeout="${BACKEND_TEST_TIMEOUT:-300s}" -covermode=atomic -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
+        golangci-lint run --config ../.golangci.yml ./... ; \
+    else echo "skip backend tests and linter" ; fi
 
+# if DRONE presented use DRONE_* git env to make version
+RUN \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="$(/script/git-rev.sh)" ; \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S) ; fi && \
+    echo "version=$version" && \
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
-FROM node:9.4-alpine as build-frontend
+FROM node:12.16-alpine as build-frontend-deps
+
+ARG CI
+ENV HUSKY_SKIP_INSTALL=true
 
-ADD web /srv/web
 RUN apk add --no-cache --update git
-RUN \
-    cd /srv/web && \
-    npm i && npm run build && \
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci --loglevel warn
+
+FROM node:12.16-alpine as build-frontend
+
+ARG CI
+ARG SKIP_FRONTEND_TEST
+ARG NODE_ENV=production
+
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
+    if [ -z "$SKIP_FRONTEND_TEST" ] ; then npx run-p lint test check; \
+    else echo "skip frontend tests and lint" ; npm run build ; fi && \
     rm -rf ./node_modules
 
-
-FROM umputun/baseimage:app-latest
+FROM umputun/baseimage:app-v1.8.0
 
 WORKDIR /srv
 
-ADD scripts/import-disqus.sh /srv/import-disqus.sh
-ADD scripts/restore-backup.sh /srv/restore-backup.sh
-ADD scripts/migrate-data.sh /srv/migrate-data.sh
-ADD scripts/create-backup.sh /srv/create-backup.sh
+ADD docker-init.sh /entrypoint.sh
+ADD backend/scripts/backup.sh /usr/local/bin/backup
+ADD backend/scripts/restore.sh /usr/local/bin/restore
+ADD backend/scripts/import.sh /usr/local/bin/import
+RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import
 
-ADD start.sh /srv/start.sh
-
-RUN chmod +x /srv/start.sh /srv/import-disqus.sh /srv/restore-backup.sh /srv/migrate-data.sh /srv/create-backup.sh
-
-COPY --from=build-backend /go/src/github.com/umputun/remark/remark /srv/
-COPY --from=build-frontend /srv/web/public/ /srv/web
+COPY --from=build-backend /build/backend/remark42 /srv/remark42
+COPY --from=build-backend /build/backend/templates /srv
+COPY --from=build-frontend /srv/frontend/public/ /srv/web
+COPY docker-init.sh /srv/init.sh
 RUN chown -R app:app /srv
+RUN ln -s /srv/remark42 /usr/bin/remark42
 
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
 
-CMD ["/srv/start.sh"]
-ENTRYPOINT ["/init.sh"]
+RUN chmod +x /srv/init.sh
+CMD ["/srv/remark42", "server"]

Dockerfile.artifacts

@@ -0,0 +1,87 @@
+FROM node:12.16-alpine as build-frontend-deps
+
+ARG CI
+ARG DRONE
+ARG DRONE_TAG
+ARG DRONE_COMMIT
+ARG DRONE_BRANCH
+
+ENV SKIP_FRONTEND_TEST=true
+
+RUN apk add --no-cache --update git
+ADD frontend/package.json /srv/frontend/package.json
+ADD frontend/package-lock.json /srv/frontend/package-lock.json
+RUN cd /srv/frontend && CI=true npm ci
+
+FROM node:12.16-alpine as build-frontend
+
+ARG CI
+ARG NODE_ENV=production
+ENV SKIP_FRONTEND_TEST=true
+ENV HUSKY_SKIP_INSTALL=true
+
+COPY --from=build-frontend-deps /srv/frontend/node_modules /srv/frontend/node_modules
+ADD frontend /srv/frontend
+RUN cd /srv/frontend && \
+    npm run build && \
+    rm -rf ./node_modules
+
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+ARG GITHUB_TOKEN
+ENV SKIP_BACKEND_TEST=true
+
+WORKDIR /build/backend
+ADD backend /build/backend
+ADD README.md /build/
+ADD LICENSE /build/
+
+ADD .git/ /build/backend/.git/
+
+COPY --from=build-frontend /srv/frontend/public/ web
+
+RUN \
+    export WEB_ROOT=/build/backend/web && \
+    find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \
+    statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \
+    statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \
+    ls -la /build/backend/app/templates/statik.go && \
+    ls -la /build/backend/app/rest/api/statik.go && \
+    ls -la /build/backend/web/
+
+# if DRONE presented use DRONE_* git env to make version
+RUN \
+    if [ -z "$DRONE" ] ; then \
+    echo "runs outside of drone" && version=$("/script/version.sh"); \
+    else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
+    echo "version=$version" && \
+    export GOFLAGS="-mod=vendor" && \
+    GOOS=linux GOARCH=amd64 go build -o remark42.linux-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=386 go build -o remark42.linux-386 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm go build -o remark42.linux-arm -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=linux GOARCH=arm64 go build -o remark42.linux-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=windows GOARCH=amd64 go build -o remark42.windows-amd64.exe -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=darwin GOARCH=amd64 go build -o remark42.darwin-amd64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=darwin GOARCH=arm64 go build -o remark42.darwin-arm64 -ldflags "-X main.revision=${version} -s -w" ./app && \
+    GOOS=freebsd GOARCH=amd64 go build -o remark42.freebsd-amd64 -ldflags "-X main.revision=${version} -s -w" ./app
+
+RUN \
+    if [ -z "$DRONE_TAG" ] ; then \
+    echo "runs outside of drone" && tag=""; \
+    else tag=_${DRONE_TAG}; fi && \
+    apk add --no-cache --update zip && \
+    cp ../LICENSE ./LICENSE && cp ../README.md ./README.md && \
+    tar cvzf remark42${tag}.linux-amd64.tar.gz remark42.linux-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-386.tar.gz remark42.linux-386 LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm.tar.gz remark42.linux-arm LICENSE README.md && \
+    tar cvzf remark42${tag}.linux-arm64.tar.gz remark42.linux-arm64 LICENSE README.md && \
+    tar cvzf remark42${tag}.darwin-amd64.tar.gz remark42.darwin-amd64 LICENSE README.md && \
+    tar cvzf remark42${tag}.darwin-arm64.tar.gz remark42.darwin-arm64 LICENSE README.md && \
+    tar cvzf remark42${tag}.freebsd-amd64.tar.gz remark42.freebsd-amd64 LICENSE README.md && \
+    zip remark42${tag}.windows-amd64.zip remark42.windows-amd64.exe LICENSE README.md
+
+
+FROM alpine
+COPY --from=build-backend /build/backend/remark42.* /artifacts/
+RUN ls -la /artifacts/*
+CMD ["sleep", "100"]

Gopkg.lock

@@ -1,237 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  name = "github.com/PuerkitoBio/goquery"
-  packages = ["."]
-  revision = "a86ea073017a6beddef78c8659e7224e8ca634b0"
-  version = "v1.4.0"
-
-[[projects]]
-  name = "github.com/andybalholm/cascadia"
-  packages = ["."]
-  revision = "901648c87902174f774fac311d7f176f8647bdaa"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/coreos/bbolt"
-  packages = ["."]
-  revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9"
-  version = "v1.3.0"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  name = "github.com/didip/tollbooth"
-  packages = [
-    ".",
-    "errors",
-    "libstring",
-    "limiter"
-  ]
-  revision = "c95eaa3ddc98f635a91e218b48727fb2e06613ea"
-  version = "v4.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/didip/tollbooth_chi"
-  packages = ["."]
-  revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
-
-[[projects]]
-  name = "github.com/go-chi/chi"
-  packages = [
-    ".",
-    "middleware"
-  ]
-  revision = "e83ac2304db3c50cf03d96a2fcd39009d458bc35"
-  version = "v3.3.2"
-
-[[projects]]
-  name = "github.com/go-chi/render"
-  packages = ["."]
-  revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/google/uuid"
-  packages = ["."]
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  name = "github.com/gorilla/feeds"
-  packages = ["."]
-  revision = "6edcbcd2d57fd0bbd7f39947a593ed0c06648388"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  revision = "b7773ae218740a7be65057fc60b366a49b538a44"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru"
-  ]
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-  packages = ["."]
-  revision = "0dc08b1671f34c4250ce212759ebd880f743d883"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"
-  packages = ["."]
-  revision = "1c38ed7ad0cc3d9e66649ac398c30e45f395c4eb"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-  packages = ["."]
-  revision = "542fd4642604d0d0c26112396ce5b1a9d01eee0b"
-
-[[projects]]
-  name = "github.com/patrickmn/go-cache"
-  packages = ["."]
-  revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0"
-  version = "v2.1.0"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/shurcooL/sanitized_anchor_name"
-  packages = ["."]
-  revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/image"
-  packages = [
-    "draw",
-    "math/f64"
-  ]
-  revision = "af66defab954cb421ca110193eed9477c8541e2a"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "html",
-    "html/atom"
-  ]
-  revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "facebook",
-    "github",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-    "yandex"
-  ]
-  revision = "543e37812f10c46c622c9575afd7ad22f22a12ba"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/time"
-  packages = ["rate"]
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch"
-  ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  packages = ["."]
-  revision = "cadec560ec52d93835bf2f15bd794700d3a2473b"
-  version = "v2.0.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "af8b7f1817ce6e82746722a745184bd50c341733bcc024311aa956ea32475796"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,36 +0,0 @@
-required = ["github.com/patrickmn/go-cache"]
-
-[[constraint]]
-  name = "github.com/go-chi/chi"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/go-chi/render"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "0.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/hashicorp/logutils"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/microcosm-cc/bluemonday"
-
-[[constraint]]
-  name = "gopkg.in/russross/blackfriday.v2"
-  version = "2.0.0"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-
-[[constraint]]
-  name = "github.com/patrickmn/go-cache"
-  version = "2.1.0"
-[[constraint]]
-  branch = "master"
-  name = "github.com/jessevdk/go-flags"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Umputun
+Copyright (c) 2021 Umputun
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,46 @@
+OS=linux
+ARCH=amd64
+
+bin:
+	docker build -f Dockerfile.artifacts -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.$(OS)-$(ARCH) remark42
+	docker rm -f remark42.bin
+
+docker:
+	docker build -t umputun/remark42 --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true .
+
+dockerx:
+	docker buildx build --build-arg SKIP_FRONTEND_TEST=true --build-arg SKIP_BACKEND_TEST=true \
+		  --progress=plain --platform linux/amd64,linux/arm/v7,linux/arm64 \
+		  -t ghcr.io/umputun/remark42:master -t umputun/remark42:master .
+
+release:
+	docker build -f Dockerfile.artifacts --no-cache --pull -t remark42.bin .
+	- @docker rm -f remark42.bin 2>/dev/null || exit 0
+	- @mkdir -p bin
+	docker run -d --name=remark42.bin remark42.bin
+	docker cp remark42.bin:/artifacts/remark42.linux-amd64.tar.gz bin/remark42.linux-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-386.tar.gz bin/remark42.linux-386.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.linux-arm64.tar.gz bin/remark42.linux-arm64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.darwin-amd64.tar.gz bin/remark42.darwin-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.freebsd-amd64.tar.gz bin/remark42.freebsd-amd64.tar.gz
+	docker cp remark42.bin:/artifacts/remark42.windows-amd64.zip bin/remark42.windows-amd64.zip
+	docker rm -f remark42.bin
+
+race_test:
+	cd backend/app && go test -race -mod=vendor -timeout=60s -count 1 ./...
+
+backend:
+	docker-compose -f compose-dev-backend.yml build
+
+frontend:
+	docker-compose -f compose-dev-frontend.yml build
+
+rundev:
+	docker pull umputun/baseimage:buildgo-latest
+	SKIP_BACKEND_TEST=true SKIP_FRONTEND_TEST=true docker-compose -f compose-private.yml build
+	docker-compose -f compose-private.yml up
+
+.PHONY: bin backend

README.md

@@ -1,481 +1,43 @@
-# remark42 [![Build Status](https://travis-ci.org/umputun/remark.svg?branch=master)](https://travis-ci.org/umputun/remark) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark)](https://goreportcard.com/report/github.com/umputun/remark)  [![Coverage Status](https://coveralls.io/repos/github/umputun/remark/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark?branch=master)
+# Remark42 [![Build Status](https://github.com/umputun/remark42/workflows/build/badge.svg)](https://github.com/umputun/remark42/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/umputun/remark42)](https://goreportcard.com/report/github.com/umputun/remark42) [![Coverage Status](https://coveralls.io/repos/github/umputun/remark42/badge.svg?branch=master)](https://coveralls.io/github/umputun/remark42?branch=master) [![codecov](https://codecov.io/gh/umputun/remark42/branch/master/graph/badge.svg)](https://app.codecov.io/gh/umputun/remark42)
 
-Remark42 is a self-hosted, lightweight, and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles or any other place where readers add comments.
+Remark42 is a self-hosted, lightweight and simple (yet functional) comment engine, which doesn't spy on users. It can be embedded into blogs, articles, or any other place where readers add comments.
 
-* Social login via Google, Facebook, Github and Yandex
+* Social login via Google, Twitter, Facebook, Microsoft, GitHub, Yandex, Patreon and Telegram
+* Login via email
+* Optional anonymous access
 * Multi-level nested comments with both tree and plain presentations
-* Import from disqus
-* Markdown support
+* Import from Disqus and WordPress
+* Markdown support with friendly formatter toolbar
 * Moderator can remove comments and block users
 * Voting, pinning and verification system
 * Sortable comments
+* Images upload with drag-and-drop
 * Extractor for recent comments, cross-post
 * RSS for all comments and each post
-* Export data to json with automatic backups
+* Telegram, Slack, Webhook and email notifications for Admins (get notified for each new comment)
+* Email and Telegram notifications for users (get notified when someone responds to your comment)
+* Export data to JSON with automatic backups
 * No external databases, everything embedded in a single data file
 * Fully dockerized and can be deployed in a single command
-* Clean, lightweight and fully customizable UI
+* Self-contained executable can be deployed directly to Linux, Windows and macOS
+* Clean, lightweight and customizable UI with white and dark themes
 * Multi-site mode from a single instance
-* Integration with automatic ssl via [nginx-le](https://github.com/umputun/nginx-le)
+* Integration with automatic SSL (direct and via [nginx-le](https://github.com/nginx-le/nginx-le))
 * [Privacy focused](#privacy)
 
-## Install
+[Demo site](https://remark42.com/demo/) available with all authentication methods, including email auth and anonymous access.
 
-### Backend
+<details><summary>Screenshots</summary>
 
-* copy provided `docker-compose.yml` and customize for your needs
-* prepare user id for container `` export USER=`id -u $USER` ``
-* make sure you **don't keep** `DEV_PASSWD=something...` for any non-development deployments
-* pull prepared images from docker hub and start - `docker-compose pull && docker-compose up -d`
-* alternatively compile from sources - `docker-compose build && docker-compose up -d`
+Comments example:
+![](screenshots/comments.png)
 
-#### Parameters
+For admin screenshots see [Admin UI wiki](https://github.com/umputun/remark42/wiki/Admin-UI)
+</details>
 
-| Command line       | Environment        | Default               | Description                                    |
-| ------------------ | ------------------ | --------------------- | ---------------------------------------------- |
-| url                | REMARK_URL         |                       | url to remark42 server, _required_             |
-| secret             | SECRET             |                       | secret key, _required_                         |
-| store.bolt.path    | STORE_BOLT_PATH    | `./var`               | path to data directory                         |
-| store.bolt.timeout | STORE_BOLT_TIMEOUT | `30s`                 | boltdb access timeout                          |
-| site               | SITE               | `remark`              | site name(s), _multi_                          |
-| admin              | ADMIN              |                       | admin names (list of user ids), _multi_        |
-| admin-email        | ADMIN_EMAIL        | `admin@${REMARK_URL}` | admin email                                    |
-| backup             | BACKUP_PATH        | `./var/backup`        | backups location                               |
-| max-back           | MAX_BACKUP_FILES   | `10`                  | max backup files to keep                       |
-| cache.max.items    | CACHE_MAX_ITEMS    | `1000`                | max number of cached items, `0` - unlimited    |
-| cache.max.value    | CACHE_MAX_VALUE    | `65536`               | max size of cached value, `0` - unlimited      |
-| cache.max.size     | CACHE_MAX_SIZE     | `50000000`            | max size of all cached values, `0` - unlimited |
-| avatar.path        | AVATAR_FS_PATH     | `./var/avatars`       | avatars location                               |
-| avatar.rsz-lmt     | AVATAR_RSZ_LMT     | 0                     | max image size for resizing avatars on save    |
-| max-comment        | MAX_COMMENT_SIZE   | 2048                  | comment's size limit                           |
-| auth.google.cid    | AUTH_GOOGLE_CID    |                       | Google OAuth client ID                         |
-| auth.google.csec   | AUTH_GOOGLE_CSEC   |                       | Google OAuth client secret                     |
-| auth.facebook.cid  | AUTH_FACEBOOK_CID  |                       | Facebook OAuth client ID                       |
-| auth.facebook.csec | AUTH_FACEBOOK_CSEC |                       | Facebook OAuth client secret                   |
-| auth.github.cid    | AUTH_GITHUB_CID    |                       | Github OAuth client ID                         |
-| auth.github.csec   | AUTH_GITHUB_CSEC   |                       | Github OAuth client secret                     |
-| auth.yandex.cid    | AUTH_YANDEX_CID    |                       | Yandex OAuth client ID                         |
-| auth.yandex.csec   | AUTH_YANDEX_CSEC   |                       | Yandex OAuth client secret                     |
-| low-score          | LOW_SCORE          | `-5`                  | Low score threshold                            |
-| critical-score     | CRITICAL_SCORE     | `-10`                 | Critical score threshold                       |
-| img-proxy          | IMG_PROXY          | `false`               | Enable http->https proxy for images            |
-| dbg                | DEBUG              | `false`               | debug mode                                     |
-| dev-passwd         | DEV_PASSWD         |                       | password for `dev` user                        |
+All remark42 documentation is available [by the link](https://remark42.com/docs/).
 
-* command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
-* _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
-* _required_ parameters have to be presented in the environment or provided in command line
+## Related projects
 
-##### Required parameters
-
-Most of the parameters have sane defaults and don't require customization. There are only a few parameters user has to define:  
-
-1. `SECRET` - secret key, can be any long and hard-to-guess string.
-1. `REMARK_URL` - url pointing to your remark42 server, i.e. `https://demo.reamark42.com`
-2. At least one pair of `AUTH_<PROVIDER>_CID` and `AUTH_<PROVIDER>_CSEC` defining oauth2 provider(s)
-
-The minimal `docker-compose.yml` has to include all required parameters:
-
-```yaml
-version: '2'
-
-services:
-    remark42:
-        image: umputun/remark42:master
-        restart: always
-        container_name: "remark42"
-        environment:
-            - REMARK_URL=https://demo.remark42.com  # url pointing to your remark42 server
-            - SECRET=abcd-123456-xyz-$%^&           # secret key
-            - AUTH_GITHUB_CID=12345667890           # oauth2 client ID
-            - AUTH_GITHUB_CSEC=abcdefg12345678      # oauth2 client secret
-            - USER=1001                             # UID on the host machine, i.e `id -u`
-        volumes:
-            - ./var:/srv/var                        # persistent volume to store all remark42 data 
-```
-
-#### Register oauth2 providers
-
-Authentication handled by external providers. You should setup oauth2 for all (or some) of them to allow users to make comments. It is not mandatory to have all of them, but at least one should be correctly configured.
-
-##### Google Auth Provider
-
-1.  Create a new project: https://console.developers.google.com/project
-1.  Choose the new project from the top right project dropdown (only if another project is selected)
-1.  In the project Dashboard center pane, choose **"API Manager"**
-1.  In the left Nav pane, choose **"Credentials"**
-1.  In the center pane, choose **"OAuth consent screen"** tab. Fill in **"Product name shown to users"** and hit save.
-1.  In the center pane, choose **"Credentials"** tab.
-    * Open the **"New credentials"** drop down
-    * Choose **"OAuth client ID"**
-    * Choose **"Web application"**
-    * Application name is freeform, choose something appropriate
-    * Authorized origins is your domain ex: `https://remark42.mysite.com`
-    * Authorized redirect URIs is the location of oauth2/callback constructed as domain + `/auth/google/callback`, ex: `https://remark42.mysite.com/auth/google/callback`
-    * Choose **"Create"**
-1.  Take note of the **Client ID** and **Client Secret**
-
-_instructions for google oauth2 setup borrowed from [oauth2_proxy](https://github.com/bitly/oauth2_proxy)_
-
-##### GitHub Auth Provider
-
-1.  Create a new **"OAuth App"**: https://github.com/settings/developers
-1.  Fill **"Application Name"** and **"Homepage URL"** for your site
-1.  Under **"Authorization callback URL"** enter the correct url constructed as domain + `/auth/github/callback`. ie `https://remark42.mysite.com/auth/github/callback`
-1.  Take note of the **Client ID** and **Client Secret**
-
-##### Facebook Auth Provider
-
-1.  From https://developers.facebook.com select **"My Apps"** / **"Add a new App"**
-1.  Set **"Display Name"** and **"Contact email"**
-1.  Choose **"Facebook Login"** and then **"Web"**
-1.  Set "Site URL" to your domain, ex: `https://remark42.mysite.com`
-1.  Under **"Facebook login"** / **"Settings"** fill "Valid OAuth redirect URIs" with your callback url constructed as domain + `/auth/facebook/callback`
-1.  Select **"App Review"** and turn public flag on. This step may ask you to provide a link to your privacy policy.
-
-##### Yandex Auth Provider
-
-1.  Create a new **"OAuth App"**: https://oauth.yandex.com/client/new
-1.  Fill **"App name"** for your site
-1.  Under **Platforms** select **"Web services"** and enter **"Callback URI #1"** constructed as domain + `/auth/yandex/callback`. ie `https://remark42.mysite.com/auth/yandex/callback`
-1.  Select **Permissions**. You need following permissions only from the **"Yandex.Passport API"** section:
-    * Access to user avatar
-    * Access to username, first name and surname, gender
-1.  Fill out the rest of fields if needed
-1.  Take note of the **ID** and **Password**
-
-For more details refer to [Yandex OAuth](https://tech.yandex.com/oauth/doc/dg/concepts/about-docpage/) and [Yandex.Passport](https://tech.yandex.com/passport/doc/dg/index-docpage/) API documentation.
-
-#### Initial import from Disqus
-
-1.  Disqus provides an export of all comments on your site in a g-zipped file. This is found in your Moderation panel at Disqus Admin > Setup > Export. The export will be sent into a queue and then emailed to the address associated with your account once it's ready. Direct link to export will be something like `https://<siteud>.disqus.com/admin/discussions/export/`. See [importing-exporting](https://help.disqus.com/customer/portal/articles/1104797-importing-exporting) for more details.
-2.  Move this file to your remark42 host within `./var` and unzip, i.e. `gunzip <disqus-export-name>.xml.gz`.
-3.  Run import command - `docker-compose exec remark42 /srv/import-disqus.sh <disqus-export-name>.xml <your site id>`
-
-#### Backup and restore
-
-##### Automatic backups
-Remark42 by default makes daily backup files under `${BACKUP_PATH}` (default `./var/backup`). Backups kept up to `${MAX_BACKUP_FILES}` (default 10). Each backup file contains exported and gzipped content, i.e., all comments. At any point, the user can restore such backup and revert all comments to the desirable state. Note: restore procedure cleans the current data store and replaces all comments with comments from the backup file.
-
-For safety and security reasons restore functionality not exposed outside of your server by default. The recommended way to restore from the backup is to use provided `scripts/restore-backup.sh`. It can run inside the container:
-
-`docker-compose exec remark42 /srv/restore-backup.sh {backup-filename.gz} {your site id}`
-
-##### Schema migration
-
-One special case for backup/restore is schema migration. Some versions or remark42 may extend or change the schema 
-and for such upgrades migration required. Provided migration script `scripts/migrate-data.sh` makes a fresh backup and then loads it back to your remark42 instance.
-
-`docker-compose exec remark42 /srv/migrate-data.sh {your site id}`
-
-##### Manual backup
-
-In addition to automatic backups user can make a backup manually. This command makes `userbackup-{site id}-{timestamp}.gz`
-
-`docker-compose exec remark42 /srv/create-backup.sh {your site id}`
-
-##### Backup format
-
-Backup file is a text file with all exported comments separated by EOL. Each backup record is a valid json with all key/value
-unmarshaled from `Comment` struct (see below). 
-
-#### Admin users
-
-Admins/moderators should be defined in `docker-compose.yml` as a list of user IDs or passed in the command line. 
-
-```
-    environment:
-        - ADMIN=github_ef0f706a79cc24b17bbbb374cd234a691a034128,github_dae9983158e9e5e127ef2b87a411ef13c891e9e5
-```
-
-To get user id just login and click on your username or any other user you want to promote to admins. 
-It will expand login info and show full user ID.
-
-
-### Frontend
-
-Frontend part is building automatically along with backend if you use `docker-compose`.
-
-For manual building:
-
-* install [Node.js 8](https://nodejs.org/en/) or higher;
-* run `npm install` inside `./web`;
-* run `npm run build` there;
-* result files will be saved in `./web/public`.
-
-For development mode use `npm start` instead of `npm run build`.
-In this case `webpack` will serve files using `webpack-dev-server` on `localhost:8080`.
-
-URLs for development:
-
-* `localhost:8080` — page with embedded script from `REMARK_URL` (default: `https://demo.remark42.com`);
-* `localhost:8080/dev.html` — page with embedded script from local folder;
-* `localhost:8080/last-comments.html` — page with embedded script for last comments;
-* `localhost:8080/counter.html` — page with embedded script for counter with examples.
-
-#### Usage
-
-##### Comments
-
-It's a main widget which renders list of comments. 
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID',
-    url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
-    max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used 
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add this node in the place where you want to see Remark42 widget:
-
-```html
-<div id="remark42"></div>
-``` 
-
-After that widget will be rendered inside this node.
-
-##### Last comments
-
-It's a widget which renders list of last comments from your site.
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add this node in the place where you want to see last comments widget:
-
-```html
-<div class="remark42__last-comments" data-max="50"></div>
-```
-
-`data-max` sets the max amount of comments (default: `15`).
-
-##### Counter
-
-It's a widget which renders a number of comments for the specified page.
-
-Add this snippet to the bottom of web page:
-
-```html
-<script>
-  var remark_config = {
-    site_id: 'YOUR_SITE_ID', 
-  };
-
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
-</script>
-```
-
-And then add a node like this in the place where you want to see a number of comments:
-
-```html
-<span class="remark42__counter" data-url="https://domain.com/path/to/article/"></span>
-```
-
-You can use as many nodes like this as you need to. 
-The script will found all them by the class `remark__counter`, 
-and it will use `data-url` attribute to define the page with comments.
-
-Also script can uses `url` property from `remark_config` object, or `window.location.href` if nothing else is defined. 
-
-## API
-
-### Authorization
-
-* `GET /auth/{provider}/login?from=http://url&session=1` - perform "social" login with one of supported providers and redirect to `url`. Presence of `session` (any non-zero value) change the default cookie expiration and makes them session-only.
-* `GET /auth/logout` - logout
-
-```go
-type User struct {
-    Name    string `json:"name"`
-    ID      string `json:"id"`
-    Picture string `json:"picture"`
-    Admin   bool   `json:"admin"`
-    Blocked bool   `json:"block"`
-    Verified bool  `json:"verified"`
-}
-```
-
-_currently supported providers are `google`, `facebook`, `github` and `yandex`_
-
-### Commenting
-
-* `POST /api/v1/comment` - add a comment. _auth required_
-
-```go
-type Comment struct {
-    ID        string          `json:"id"`      // comment ID, read only
-    ParentID  string          `json:"pid"`     // parent ID
-    Text      string          `json:"text"`    // comment text, after md processing
-    Orig      string          `json:"orig"`    // original comment text
-    User      User            `json:"user"`    // user info, read only
-    Locator   Locator         `json:"locator"` // post locator
-    Score     int             `json:"score"`   // comment score, read only
-    Votes     map[string]bool `json:"votes"`   // comment votes, read only
-    Timestamp time.Time       `json:"time"`    // time stamp, read only
-    Pin       bool            `json:"pin"`     // pinned status, read only
-    Delete    bool            `json:"delete"`  // delete status, read only
-}
-
-type Locator struct {
-    SiteID string `json:"site"`     // site id
-    URL    string `json:"url"`      // post url
-}
-```
-
-* `POST /api/v1/preview` - preview comment in html. Body is `Comment` to render
-
-* `GET /api/v1/find?site=site-id&url=post-url&sort=fld&format=tree|plain` - find all comments for given post
-
-This is the primary call used by UI to show comments for given post. It can return comments in two formats - `plain` and `tree`.
-In plain format result will be sorted list of `Comment`. In tree format this is going to be tree-like object with this structure:
-
-```go
-type Tree struct {
-    Nodes []Node `json:"comments"`
-    Info  store.PostInfo `json:"info,omitempty"`
-}
-
-type Node struct {
-    Comment store.Comment `json:"comment"`
-    Replies []Node        `json:"replies,omitempty"`
-}
-```
-
-Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i.e. `-time`. For `tree` mode sort will be applied to top-level comments only and all replies always sorted by time.
-
-* `PUT /api/v1/comment/{id}?site=site-id&url=post-url` - edit comment, allowed once in 5min since creation
-
-```json
-  Content-Type: application/json
-
-  {
-    "text": "edit comment blah http://radio-t.com 12345",
-    "summary": "fix blah"
-  }
-```
-
-* `GET /api/v1/last/{max}?site=site-id` - get up to `{max}` last comments
-* `GET /api/v1/id/{id}?site=site-id` - get comment by `comment id`
-* `GET /api/v1/comments?site=site-id&user=id&limit=N` - get comment by `user id`, returns `response` object
-  ```go
-  type response struct {
-      Comments []store.Comment  `json:"comments"`
-      Count    int              `json:"count"`
-  }{}
-  ```
-* `GET /api/v1/count?site=site-id&url=post-url` - get comment's count for `{url}`
-* `GET /api/v1/list?site=site-id&limit=5&skip=2` - list commented posts, returns array or `PostInfo`, limit=0 will return all posts
-  ```go
-  type PostInfo struct {
-      URL   string      `json:"url"`
-      Count int         `json:"count"`
-      ReadOnly bool     `json:"read_only,omitempty"`
-      FirstTS time.Time `json:"first_time,omitempty"`
-      LastTS  time.Time `json:"last_time,omitempty"`
-  }
-  ```
-* `GET /api/v1/user` - get user info, _auth required_
-* `PUT /api/v1/vote/{id}?site=site-id&url=post-url&vote=1` - vote for comment. `vote`=1 will increase score, -1 decrease. _auth required_
-* `GET /api/v1/userdata?site=site-id` - export all user data to gz stream  _auth required_
-* `POST /api/v1/deleteme?site=site-id` - request deletion of user data. _auth required_
-* `GET /api/v1/config?site=site-id` - returns configuration (parameters) for given site
-
-  ```go
-  type config struct {
-      Version       string   `json:"version"`
-      EditDuration  int      `json:"edit_duration"` // seconds
-      Admins        []string `json:"admins"`
-      Auth          []string `json:"auth_providers"`
-      LowScore      int      `json:"low_score"`
-      CriticalScore int      `json:"critical_score"`
-  }
-  ``` 
-* `GET /api/v1/info?site=site-idd&url=post-ur` - returns `PostInfo` for site and url
-  
-### RSS feeds
-  
-* `GET /api/v1/rss/post?site=site-id&url=post-url` - rss feed for a post
-* `GET /api/v1/rss/site?site=site-id` - rss feed for given site
-
-### Admin
-
-* `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.
-* `PUT /api/v1/admin/user/{userid}?site=site-id&block=1` - block or unblock user.
-* `GET api/v1/admin/blocked&site=site-id` - list of blocked user ids.
-  ```go
-  type BlockedUser struct {
-      ID        string    `json:"id"`
-      Name      string    `json:"name"`
-      Timestamp time.Time `json:"time"`
-  }
-  ```
-* `GET /api/v1/admin/export?site=side-id&mode=[stream|file]` - export all comments to json stream or gz file.
-* `POST /api/v1/admin/import?site=side-id` - import comments from the backup.
-* `PUT /api/v1/admin/pin/{id}?site=site-id&url=post-url&pin=1` - pin or unpin comment.
-* `DELETE /api/v1/admin/user/{userid}?site=site-id&block=1` - delete all user's comments.
-* `PUT /api/v1/admin/readonly?site=site-id&url=post-url&ro=1` - set read-only status
-* `PUT /api/v1/admin/verify/{userid}?site=site-id&verified=1` - set verified status
-* `GET /api/v1/admin/deleteme?token=token` - process deleteme user's request
-
-_all admin calls require auth and admin privilege_
-
-
-## Privacy 
-
-* Remark42 is trying to be very sensitive to any private or semi-private information.
-* Authentication requesting the lowest (minimal) possible scope from providers. All extra information returned by them dropped immediately and not stored in any form.
-* Generally remark42 keeps user id, username and avatar link only. None of these fields exposed directly - id and name hashed, avatar proxied.
-* There is no tracking of any sort.
-* Login mechanic uses JWT stored in a cookie (httpOnly, secured). The second cookie (XSRF_TOKEN) is a random id preventing Cross-Site Request Forgery
-* There is no cross-site login, i.e., user's behavior can't be analyzed across independent sites running remark42.
-* There are no third-party analytic services involved.
-* User can request all information remark42 knows about and export to gz file.
-* Supported complete cleanup of all information related to user activity on demand.
-* Cookie lifespan can be restricted to session-only. 
-* All potentially sensitive data stored by remark42 hashed and encrypted.
-
-
-## Technical details
-
-* Data stored in [boltdb](https://github.com/coreos/bbolt) (embedded key/value database) files under `BOLTDB_PATH`
-* Each site stored in a separate boltbd file.
-* In order to migrate/move remark42 to another host boltbd files as well as avatars directory `AVATAR_STORE` should be transferred.
-* Automatic backup process runs every 24h and exports all content in json-like format to `backup-remark-YYYYMMDD.gz`.
-* Authentication implemented with [jwt](https://github.com/dgrijalva/jwt-go) stored in a cookie. It uses HttpOnly, secure cookies.
-* All heavy REST calls cached internally in LRU cache limited by `MAX_CACHE_ITEMS` and `MAX_CACHE_SIZE`.
-* User's activity throttled globally (up to 1000 simultaneous requests) and limited locally (per user, usually up to 10 req/sec)
-* Request timeout set to 60sec
-* Development mode (`--dev-password` set) allows to test remark42 without social login and with admin privileges. Adds basic-auth for username: `dev`, password: `${DEV_PASSWD}`. **should not be used in production deployment**
-* User can vote for the comment multiple times but only to change his/her vote. Double-voting not allowed.
-* User can edit comments in 5 mins window after creation.
-* User ID hashed and prefixed by oauth provider name to avoid collisions and potential abuse.
-* All avatars cached locally to prevent rate limiters from google/github/facebook/yandex.
-* Docker build uses [publicly available](https://github.com/umputun/baseimage) base images.
+* [A Helm chart for Remark42 on Kubernetes](https://github.com/groundhog2k/helm-charts/tree/master/charts/remark42)
+* [django-remark42](https://github.com/andrewp-as-is/django-remark42.py)

SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+We release patches for security vulnerabilities.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| current | :white_check_mark: |
+| >=1.6.x | :white_check_mark: |
+| <1.5.x  | :x:                |
+
+## Reporting a Vulnerability
+
+Please report (suspected) security vulnerabilities to umputun@gmail.com. You will receive a response from us within 48 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity but historically within a few days.

app/main.go

@@ -1,398 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/hashicorp/logutils"
-	"github.com/jessevdk/go-flags"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/api"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Opts with command line flags and env
-// nolint:maligned
-type Opts struct {
-	SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
-	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
-
-	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
-	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
-	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-
-	Sites          []string `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	Admins         []string `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
-	AdminEmail     string   `long:"admin-email" env:"ADMIN_EMAIL" default:"" description:"admin email"`
-	DevPasswd      string   `long:"dev-passwd" env:"DEV_PASSWD" default:"" description:"development mode password"`
-	BackupLocation string   `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles int      `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	ImageProxy     bool     `long:"img-proxy" env:"IMG_PROXY" description:"enable image proxy"`
-	MaxCommentSize int      `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	LowScore       int      `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore  int      `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	ReadOnlyAge    int      `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments"`
-	Port           int      `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	WebRoot        string   `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	Dbg            bool     `long:"dbg" env:"DEBUG" description:"debug mode"`
-
-	Auth struct {
-		Google   AuthGroup `group:"google" namespace:"google" env-namespace:"GOOGLE" description:"Google OAuth"`
-		Github   AuthGroup `group:"github" namespace:"github" env-namespace:"GITHUB" description:"Github OAuth"`
-		Facebook AuthGroup `group:"facebook" namespace:"facebook" env-namespace:"FACEBOOK" description:"Facebook OAuth"`
-		Yandex   AuthGroup `group:"yandex" namespace:"yandex" env-namespace:"YANDEX" description:"Yandex OAuth"`
-	} `group:"auth" namespace:"auth" env-namespace:"AUTH"`
-}
-
-// AuthGroup defines options group for auth params
-type AuthGroup struct {
-	CID  string `long:"cid" env:"CID" description:"OAuth client ID"`
-	CSEC string `long:"csec" env:"CSEC" description:"OAuth client secret"`
-}
-
-// StoreGroup defines options group for store params
-type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
-	Bolt struct {
-		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
-		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
-	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
-}
-
-// AvatarGroup defines options group for avatar params
-type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"mongo" default:"fs"`
-	FS   struct {
-		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
-	} `group:"fs" namespace:"fs" env-namespace:"FS"`
-	RszLmt int `long:"rsz-lmt" env:"RSZ_LMT" default:"0" description:"max image size for resizing avatars on save"`
-}
-
-// CacheGroup defines options group for cache params
-type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
-	Max  struct {
-		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
-		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
-		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
-	} `group:"max" namespace:"max" env-namespace:"MAX"`
-}
-
-var revision = "unknown"
-
-// Application holds all active objects
-type Application struct {
-	Opts
-	restSrv     *api.Rest
-	migratorSrv *api.Migrator
-	exporter    migrator.Exporter
-	terminated  chan struct{}
-}
-
-func main() {
-	fmt.Printf("remark %s\n", revision)
-
-	var opts Opts
-	p := flags.NewParser(&opts, flags.Default)
-	if _, e := p.ParseArgs(os.Args[1:]); e != nil {
-		os.Exit(1)
-	}
-
-	setupLog(opts.Dbg)
-	log.Print("[INFO] started remark")
-	resetEnv("SECRET", "AUTH_GOOGLE_CSEC", "AUTH_GITHUB_CSEC", "AUTH_FACEBOOK_CSEC", "AUTH_YANDEX_CSEC")
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() { // catch signal and invoke graceful termination
-		stop := make(chan os.Signal, 1)
-		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
-		<-stop
-		log.Print("[WARN] interrupt signal")
-		cancel()
-	}()
-
-	app, err := New(opts)
-	if err != nil {
-		log.Fatalf("[ERROR] failed to setup application, %+v", err)
-	}
-	err = app.Run(ctx)
-	log.Printf("[INFO] remark terminated %s", err)
-}
-
-// New prepares application and return it with all active parts
-// doesn't start anything
-func New(opts Opts) (*Application, error) {
-
-	if err := makeDirs(opts.BackupLocation); err != nil {
-		return nil, err
-	}
-
-	if !strings.HasPrefix(opts.RemarkURL, "http://") && !strings.HasPrefix(opts.RemarkURL, "https://") {
-		return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
-	}
-
-	boltStore, err := makeDataStore(opts.Store, opts.Sites)
-	if err != nil {
-		return nil, err
-	}
-	dataService := &service.DataStore{
-		Interface:      boltStore,
-		EditDuration:   5 * time.Minute,
-		Secret:         opts.SecretKey,
-		MaxCommentSize: opts.MaxCommentSize,
-	}
-
-	loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
-		cache.MaxKeys(opts.Cache.Max.Items), cache.PostFlushFn(postFlushFn(opts.Sites, opts.Port)))
-	if err != nil {
-		return nil, err
-	}
-
-	jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"), 7*24*time.Hour)
-
-	avatarStore, err := makeAvatarStore(opts.Avatar)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make avatar store")
-	}
-	avatarProxy := &proxy.Avatar{
-		Store:     avatarStore,
-		RoutePath: "/api/v1/avatar",
-		RemarkURL: strings.TrimSuffix(opts.RemarkURL, "/"),
-	}
-
-	exporter := &migrator.Remark{DataStore: dataService}
-
-	migr := &api.Migrator{
-		Version:        revision,
-		Cache:          loadingCache,
-		NativeImporter: &migrator.Remark{DataStore: dataService},
-		DisqusImporter: &migrator.Disqus{DataStore: dataService},
-		NativeExported: &migrator.Remark{DataStore: dataService},
-		SecretKey:      opts.SecretKey,
-	}
-
-	srv := &api.Rest{
-		Version:     revision,
-		DataService: dataService,
-		Exporter:    exporter,
-		WebRoot:     opts.WebRoot,
-		RemarkURL:   opts.RemarkURL,
-		ImageProxy:  &proxy.Image{Enabled: opts.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: opts.RemarkURL},
-		AvatarProxy: avatarProxy,
-		ReadOnlyAge: opts.ReadOnlyAge,
-		Authenticator: auth.Authenticator{
-			JWTService: jwtService,
-			Admins:     opts.Admins,
-			AdminEmail: opts.AdminEmail,
-			Providers:  makeAuthProviders(jwtService, avatarProxy, dataService, opts),
-			DevPasswd:  opts.DevPasswd,
-		},
-		Cache: loadingCache,
-	}
-
-	// no admin email, use admin@domain
-	if srv.Authenticator.AdminEmail == "" {
-		if u, err := url.Parse(opts.RemarkURL); err == nil {
-			srv.Authenticator.AdminEmail = "admin@" + u.Host
-		}
-	}
-
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = opts.LowScore, opts.CriticalScore
-	tch := make(chan struct{})
-	return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, Opts: opts, terminated: tch}, nil
-}
-
-// Run all application objects
-func (a *Application) Run(ctx context.Context) error {
-	if a.DevPasswd != "" {
-		log.Printf("[WARN] running in dev mode")
-	}
-
-	go func() {
-		// shutdown on context cancellation
-		<-ctx.Done()
-		a.restSrv.Shutdown()
-		a.migratorSrv.Shutdown()
-	}()
-	a.activateBackup(ctx) // runs in goroutine for each site
-	go a.migratorSrv.Run(a.Port + 1)
-	a.restSrv.Run(a.Port)
-	close(a.terminated)
-	return nil
-}
-
-// Wait for application completion (termination)
-func (a *Application) Wait() {
-	<-a.terminated
-}
-
-// activateBackup runs background backups for each site
-func (a *Application) activateBackup(ctx context.Context) {
-	for _, siteID := range a.Sites {
-		backup := migrator.AutoBackup{
-			Exporter:       a.exporter,
-			BackupLocation: a.BackupLocation,
-			SiteID:         siteID,
-			KeepMax:        a.MaxBackupFiles,
-			Duration:       24 * time.Hour,
-		}
-		go backup.Do(ctx)
-	}
-}
-
-// makeDataStore creates store for all sites
-func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
-	switch group.Type {
-	case "bolt":
-		if err = makeDirs(group.Bolt.Path); err != nil {
-			return nil, err
-		}
-		sites := []engine.BoltSite{}
-		for _, site := range siteNames {
-			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
-		}
-		result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
-	default:
-		return nil, errors.Errorf("unsupported store type %s", group.Type)
-	}
-
-	return result, errors.Wrap(err, "can't initialize data store")
-}
-
-func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
-	switch group.Type {
-	case "fs":
-		if err = makeDirs(group.FS.Path); err != nil {
-			return nil, err
-		}
-		return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
-	}
-	return nil, errors.Errorf("unsupported avatart store type %s", group.Type)
-}
-
-// mkdir -p for all dirs
-func makeDirs(dirs ...string) error {
-
-	// exists returns whether the given file or directory exists or not
-	exists := func(path string) (bool, error) {
-		_, err := os.Stat(path)
-		if err == nil {
-			return true, nil
-		}
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return true, err
-	}
-
-	for _, dir := range dirs {
-		ex, err := exists(dir)
-		if err != nil {
-			return errors.Wrapf(err, "can't check directory status for %s", dir)
-		}
-		if !ex {
-			if e := os.MkdirAll(dir, 0700); e != nil {
-				return errors.Wrapf(err, "can't make directory %s", dir)
-			}
-		}
-	}
-	return nil
-}
-
-func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
-
-	makeParams := func(cid, secret string) auth.Params {
-		return auth.Params{
-			JwtService:   jwtService,
-			AvatarProxy:  avatarProxy,
-			RemarkURL:    opts.RemarkURL,
-			Cid:          cid,
-			Csecret:      secret,
-			Admins:       opts.Admins,
-			SecretKey:    opts.SecretKey,
-			IsVerifiedFn: ds.IsVerifiedFn(),
-		}
-	}
-
-	providers := []auth.Provider{}
-	if opts.Auth.Google.CID != "" && opts.Auth.Google.CSEC != "" {
-		providers = append(providers, auth.NewGoogle(makeParams(opts.Auth.Google.CID, opts.Auth.Google.CSEC)))
-	}
-	if opts.Auth.Github.CID != "" && opts.Auth.Github.CSEC != "" {
-		providers = append(providers, auth.NewGithub(makeParams(opts.Auth.Github.CID, opts.Auth.Github.CSEC)))
-	}
-	if opts.Auth.Facebook.CID != "" && opts.Auth.Facebook.CSEC != "" {
-		providers = append(providers, auth.NewFacebook(makeParams(opts.Auth.Facebook.CID, opts.Auth.Facebook.CSEC)))
-	}
-	if opts.Auth.Yandex.CID != "" && opts.Auth.Yandex.CSEC != "" {
-		providers = append(providers, auth.NewYandex(makeParams(opts.Auth.Yandex.CID, opts.Auth.Yandex.CSEC)))
-	}
-	if len(providers) == 0 {
-		log.Printf("[WARN] no auth providers defined")
-	}
-	return providers
-}
-
-// post-flush callback invoked by cache after each flush in async way
-func postFlushFn(sites []string, port int) func() {
-
-	return func() {
-		// list of heavy urls for pre-heating on cache change
-		urls := []string{
-			"http://localhost:%d/api/v1/list?site=%s",
-			"http://localhost:%d/api/v1/last/50?site=%s",
-		}
-
-		for _, site := range sites {
-			for _, u := range urls {
-				resp, err := http.Get(fmt.Sprintf(u, port, site))
-				if err != nil {
-					log.Printf("[WARN] failed to refresh cached list for %s, %s", site, err)
-					return
-				}
-				if err = resp.Body.Close(); err != nil {
-					log.Printf("[WARN] failed to close response body, %s", err)
-				}
-			}
-		}
-	}
-}
-
-func resetEnv(envs ...string) {
-	for _, env := range envs {
-		if err := os.Unsetenv(env); err != nil {
-			log.Printf("[WARN] can't unset env %s, %s", env, err)
-		}
-	}
-}
-
-func setupLog(dbg bool) {
-	filter := &logutils.LevelFilter{
-		Levels:   []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR"},
-		MinLevel: logutils.LogLevel("INFO"),
-		Writer:   os.Stdout,
-	}
-
-	log.SetFlags(log.Ldate | log.Ltime)
-
-	if dbg {
-		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)
-		filter.MinLevel = logutils.LogLevel("DEBUG")
-	}
-	log.SetOutput(filter)
-}

app/main_test.go

@@ -1,135 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	flags "github.com/jessevdk/go-flags"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestApplication(t *testing.T) {
-	app, ctx := prepApp(t, 18080, 500*time.Millisecond)
-	go func() { _ = app.Run(ctx) }()
-	time.Sleep(100 * time.Millisecond) // let server start
-
-	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
-
-	// add comment
-	resp, err = http.Post("http://dev:password@localhost:18080/api/v1/comment", "json",
-		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
-	require.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	body, _ = ioutil.ReadAll(resp.Body)
-	t.Log(string(body))
-
-	assert.Equal(t, "admin@demo.remark42.com", app.restSrv.Authenticator.AdminEmail, "default admin email")
-
-	app.Wait()
-}
-
-func TestApplicationFailed(t *testing.T) {
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-
-	// RO bolt location
-	_, err := p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/dev/null"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't initialize data store: failed to make boltdb for /dev/null/remark.db: "+
-		"open /dev/null/remark.db: not a directory")
-	t.Log(err)
-
-	// RO backup location
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.bolt.path=/tmp",
-		"--backup=/dev/null/not-writable"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "can't check directory status for /dev/null/not-writable: stat /dev/null/not-writable: not a directory")
-	t.Log(err)
-
-	// invalid url
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=demo.remark42.com", "----store.bolt.path=/tmp"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "invalid remark42 url demo.remark42.com")
-	t.Log(err)
-
-	opts = Opts{}
-	_, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--store.type=mongo"})
-	assert.Nil(t, err)
-	_, err = New(opts)
-	assert.EqualError(t, err, "unsupported store type mongo")
-	t.Log(err)
-}
-
-func TestApplicationShutdown(t *testing.T) {
-	app, ctx := prepApp(t, 18090, 500*time.Millisecond)
-	st := time.Now()
-	err := app.Run(ctx)
-	assert.Nil(t, err)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-	app.Wait()
-}
-
-func TestApplicationMainSignal(t *testing.T) {
-	os.Args = []string{"test", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp", "--avatar.fs.path=/tmp",
-		"--port=18100", "--url=https://demo.remark42.com"}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
-	}()
-	st := time.Now()
-	main()
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
-}
-
-func prepApp(t *testing.T, port int, duration time.Duration) (*Application, context.Context) {
-	// prepare options
-	opts := Opts{}
-	p := flags.NewParser(&opts, flags.Default)
-	_, err := p.ParseArgs([]string{"--secret=123456", "--dev-passwd=password", "--url=https://demo.remark42.com"})
-	require.Nil(t, err)
-	opts.Avatar.FS.Path, opts.Avatar.Type, opts.BackupLocation = "/tmp", "fs", "/tmp"
-	opts.Store.Bolt.Path = fmt.Sprintf("/tmp/%d", port)
-	opts.Store.Bolt.Timeout = 10 * time.Second
-	opts.Auth.Github.CSEC, opts.Auth.Github.CID = "csec", "cid"
-	opts.Auth.Google.CSEC, opts.Auth.Google.CID = "csec", "cid"
-	opts.Auth.Facebook.CSEC, opts.Auth.Facebook.CID = "csec", "cid"
-	opts.Auth.Yandex.CSEC, opts.Auth.Yandex.CID = "csec", "cid"
-	opts.Port = port
-
-	os.Remove(opts.Store.Bolt.Path + "/remark.db")
-
-	// create app
-	app, err := New(opts)
-	require.Nil(t, err)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(duration)
-		log.Print("[TEST] terminate app")
-		cancel()
-	}()
-	return app, ctx
-}

app/migrator/backup.go

@@ -1,94 +0,0 @@
-package migrator
-
-import (
-	"compress/gzip"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/pkg/errors"
-)
-
-// AutoBackup struct handles daily backups params for siteID
-type AutoBackup struct {
-	Exporter       Exporter
-	BackupLocation string
-	SiteID         string
-	KeepMax        int
-	Duration       time.Duration
-}
-
-// Do runs daily export to local files, keeps up to keepMax backups for given siteID
-func (ab AutoBackup) Do(ctx context.Context) {
-	log.Printf("[INFO] activate auto-backup for %s under %s, duration %s", ab.SiteID, ab.BackupLocation, ab.Duration)
-	tick := time.NewTicker(ab.Duration)
-	log.Printf("[DEBUG] first backup for %s at %s", ab.SiteID, time.Now().Add(ab.Duration))
-
-	for {
-		select {
-		case <-tick.C:
-			if _, err := ab.makeBackup(); err != nil {
-				log.Printf("[WARN] auto-backup for %s failed, %s", ab.SiteID, err)
-				continue
-			}
-			ab.removeOldBackupFiles()
-			log.Printf("[DEBUG] next backup for %s at %s", ab.SiteID, time.Now().Add(ab.Duration))
-		case <-ctx.Done():
-			log.Printf("[WARN] terminated autobackup for %s", ab.SiteID)
-			return
-		}
-	}
-}
-
-func (ab AutoBackup) makeBackup() (string, error) {
-	log.Printf("[DEBUG] make backup for %s", ab.SiteID)
-	backupFile := fmt.Sprintf("%s/backup-%s-%s.gz", ab.BackupLocation, ab.SiteID, time.Now().Format("20060102"))
-	fh, err := os.Create(backupFile)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't create backup file %s", backupFile)
-	}
-	gz := gzip.NewWriter(fh)
-
-	if _, err = ab.Exporter.Export(gz, ab.SiteID); err != nil {
-		return "", errors.Wrapf(err, "export failed for %s", ab.SiteID)
-	}
-	if err = gz.Close(); err != nil {
-		return "", errors.Wrapf(err, "can't close gz for %s", backupFile)
-	}
-	if err = fh.Close(); err != nil {
-		return "", errors.Wrapf(err, "can't close file handler for %s", backupFile)
-	}
-	log.Printf("[DEBUG] created backup file %s", backupFile)
-	return backupFile, nil
-}
-
-func (ab AutoBackup) removeOldBackupFiles() {
-	files, err := ioutil.ReadDir(ab.BackupLocation)
-	if err != nil {
-		log.Printf("[WARN] can't read files in backup directory %s, %s", ab.BackupLocation, err)
-		return
-	}
-	backFiles := []os.FileInfo{}
-	for _, file := range files {
-		if strings.HasPrefix(file.Name(), "backup-"+ab.SiteID) {
-			backFiles = append(backFiles, file)
-		}
-	}
-	sort.Slice(backFiles, func(i int, j int) bool { return backFiles[i].Name() < backFiles[j].Name() })
-
-	if len(backFiles) > ab.KeepMax {
-		for i := 0; i < len(backFiles)-ab.KeepMax; i++ {
-			fpath := ab.BackupLocation + "/" + backFiles[i].Name()
-			if e := os.Remove(fpath); e != nil {
-				log.Printf("[WARN] can't delete %s, %s", fpath, err)
-				continue
-			}
-			log.Printf("[DEBUG] removed %s", fpath)
-		}
-	}
-}

app/migrator/backup_test.go

@@ -1,81 +0,0 @@
-package migrator
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBackup_RemoveOldBackupFiles(t *testing.T) {
-	loc := "/tmp/remark-backups.test"
-	defer os.RemoveAll(loc)
-
-	os.MkdirAll(loc, 0700)
-	for i := 1; i <= 10; i++ {
-		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
-		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-		assert.Nil(t, err)
-	}
-	fname := fmt.Sprintf("%s/backup-site2-20171210.gz", loc)
-	err := ioutil.WriteFile(fname, []byte("blah"), 0600)
-	assert.Nil(t, err)
-
-	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3}
-	bk.removeOldBackupFiles()
-	ff, err := ioutil.ReadDir(loc)
-	assert.Nil(t, err)
-	assert.Equal(t, 4, len(ff), "should keep 4 files - 3 kept for sit1, and one for site2")
-	assert.Equal(t, "backup-site1-20171208.gz", ff[0].Name())
-	assert.Equal(t, "backup-site1-20171209.gz", ff[1].Name())
-	assert.Equal(t, "backup-site1-20171210.gz", ff[2].Name())
-	assert.Equal(t, "backup-site2-20171210.gz", ff[3].Name())
-}
-
-func TestBackup_MakeBackup(t *testing.T) {
-	loc := "/tmp/remark-backups.test"
-	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
-
-	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
-	fname, err := bk.makeBackup()
-	assert.NoError(t, err)
-	expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
-	assert.Equal(t, expFile, fname)
-
-	fi, err := os.Lstat(expFile)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(52), fi.Size())
-}
-
-func TestBackup_Do(t *testing.T) {
-	loc := "/tmp/remark-backups.test"
-	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(time.Second)
-		cancel()
-	}()
-
-	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}, Duration: 600 * time.Millisecond}
-	bk.Do(ctx)
-
-	expFile := fmt.Sprintf("/tmp/remark-backups.test/backup-site1-%s.gz", time.Now().Format("20060102"))
-	fi, err := os.Lstat(expFile)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(52), fi.Size())
-}
-
-type mockExporter struct{}
-
-func (mock *mockExporter) Export(w io.Writer, siteID string) (int, error) {
-	_, err := w.Write([]byte("some export blah blah 1234567890"))
-	return 1000, err
-}

app/migrator/disqus.go

@@ -1,165 +0,0 @@
-package migrator
-
-import (
-	"encoding/xml"
-	"io"
-	"log"
-	"strings"
-	"time"
-
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// Disqus implements Importer from disqus xml
-type Disqus struct {
-	DataStore Store
-}
-
-type disqusThread struct {
-	UID         string    `xml:"id,attr"`
-	Forum       string    `xml:"forum"`
-	Link        string    `xml:"link"`
-	Title       string    `xml:"title"`
-	Message     string    `xml:"message"`
-	CreateAt    time.Time `xml:"createdAt"`
-	AuthorName  string    `xml:"author>name"`
-	AuthorEmail string    `xml:"author>email"`
-	IP          string    `xml:"ipAddress"`
-	Anonymous   bool      `xml:"author>isAnonymous"`
-	Closed      bool      `xml:"isClosed"`
-	Deleted     bool      `xml:"isDeleted"`
-}
-
-type disqusComment struct {
-	UID            string    `xml:"id,attr"`
-	ID             string    `xml:"id"`
-	Message        string    `xml:"message"`
-	CreatedAt      time.Time `xml:"createdAt"`
-	AuthorEmail    string    `xml:"author>email"`
-	AuthorName     string    `xml:"author>name"`
-	AuthorUserName string    `xml:"author>username"`
-	IP             string    `xml:"ipAddress"`
-	Tid            uid       `xml:"thread"`
-	Pid            uid       `xml:"parent"`
-	IsSpam         bool      `xml:"isSpam"`
-}
-
-type uid struct {
-	Val string `xml:"id,attr"`
-}
-
-// Import from disqus and save to store
-func (d *Disqus) Import(r io.Reader, siteID string) (size int, err error) {
-
-	if err = d.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
-	}
-
-	commentsCh := d.convert(r, siteID)
-	failed, passed := 0, 0
-	for c := range commentsCh {
-		if _, err = d.DataStore.Create(c); err != nil {
-			failed++
-			continue
-		}
-		passed++
-	}
-
-	if failed > 0 {
-		return passed, errors.Errorf("failed to save %d comments", failed)
-	}
-
-	log.Printf("[DEBUG] imported %d comments to site %s", passed, siteID)
-
-	if failed > 0 && passed == 0 {
-		err = errors.New("import failed")
-	}
-	return passed, err
-}
-
-// convert disqus stream (xml) from reader and fill channel of comments.
-// runs async and closes channel on completion.
-func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
-
-	postsMap := map[string]string{} // tid:url
-	decoder := xml.NewDecoder(r)
-	commentsCh := make(chan store.Comment)
-
-	stats := struct {
-		inpThreads, inpComments     int
-		commentsCount, spamComments int
-		failedThreads, failedPosts  int
-	}{}
-
-	go func() {
-		for {
-			t, err := decoder.Token()
-			if t == nil || err != nil {
-				break
-			}
-
-			switch se := t.(type) {
-			case xml.StartElement:
-				if se.Name.Local == "thread" {
-					stats.inpThreads++
-					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
-						log.Printf("[WARN] can't decode disqus thread, %s", err)
-						stats.failedThreads++
-						continue
-					}
-					postsMap[thread.UID] = thread.Link
-					continue
-				}
-				if se.Name.Local == "post" {
-					stats.inpComments++
-					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
-						log.Printf("[WARN] can't decode disqus comment, %s", err)
-						stats.failedPosts++
-						continue
-					}
-					if comment.IsSpam {
-						stats.spamComments++
-						continue
-					}
-					c := store.Comment{
-						ID:      comment.UID,
-						Locator: store.Locator{URL: postsMap[comment.Tid.Val], SiteID: siteID},
-						User: store.User{
-							ID:   "disqus_" + store.EncodeID(comment.AuthorUserName),
-							Name: comment.AuthorName,
-							IP:   comment.IP,
-						},
-						Text:      d.cleanText(comment.Message),
-						Timestamp: comment.CreatedAt,
-						ParentID:  comment.Pid.Val,
-					}
-					if c.User.ID == "disqus_" { // empty comment.AuthorUserName from disqus
-						c.User.ID = "disqus_" + c.User.Name
-					}
-					if c.ID == "" { // no comment.UID
-						c.ID = comment.ID
-					}
-					commentsCh <- c
-					stats.commentsCount++
-					if stats.commentsCount%1000 == 0 {
-						log.Printf("[DEBUG] processed %d comments", stats.commentsCount)
-					}
-				}
-			}
-		}
-		close(commentsCh)
-		log.Printf("[INFO] converted %d posts, %+v", len(postsMap), stats)
-	}()
-
-	return commentsCh
-}
-
-func (d *Disqus) cleanText(text string) string {
-	text = strings.Replace(text, "\n", "", -1)
-	text = strings.Replace(text, "\t", "", -1)
-	return text
-}

app/migrator/disqus_test.go

@@ -1,193 +0,0 @@
-package migrator
-
-import (
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestDisqus_Import(t *testing.T) {
-	defer os.Remove("/tmp/remark-test.db")
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := service.DataStore{Interface: b}
-	d := Disqus{DataStore: &dataStore}
-	size, err := d.Import(strings.NewReader(xmlTest), "test")
-	assert.Nil(t, err)
-	assert.Equal(t, 3, size)
-
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
-
-	c := last[0]
-	assert.True(t, strings.HasPrefix(c.Text, "<p>Google App Engine"))
-	assert.Equal(t, "299986072", c.ID)
-	assert.Equal(t, "", c.ParentID)
-	assert.Equal(t, store.Locator{SiteID: "test", URL: "http://radio-t.umputun.com/2011/03/229_8880.html"}, c.Locator)
-	assert.Equal(t, "Dmitry Noname", c.User.Name)
-	assert.Equal(t, "disqus_8799342cdf328253e03313958ffc6a433659d7ff", c.User.ID)
-	assert.Equal(t, "96243f024cf6ad42b66f0c72709ae20b5d10ec14", c.User.IP)
-
-	posts, err := dataStore.List("test", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(posts), "2 posts")
-
-	count, err := dataStore.Count(store.Locator{SiteID: "test", URL: "https://radio-t.com/p/2011/03/05/podcast-229/"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-}
-
-func TestDisqus_Convert(t *testing.T) {
-	d := Disqus{}
-	ch := d.convert(strings.NewReader(xmlTest), "test")
-
-	res := []store.Comment{}
-	for comment := range ch {
-		res = append(res, comment)
-	}
-	assert.Equal(t, 3, len(res), "3 comments total, 1 spam excluded")
-
-	exp0 := store.Comment{
-		ID: "299619020",
-		Locator: store.Locator{
-			SiteID: "test",
-			URL:    "https://radio-t.com/p/2011/03/05/podcast-229/",
-		},
-		Text: `<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>`,
-		User: store.User{
-			Name: "Alexander Blah",
-			ID:   "disqus_328c8b68974aef73785f6b38c3d3fedfdf941434",
-			IP:   "178.178.178.178",
-		},
-	}
-	exp0.Timestamp, _ = time.Parse("2006-01-02T15:04:05Z", "2011-08-31T15:16:29Z")
-	assert.Equal(t, exp0, res[0])
-}
-
-var xmlTest = `<?xml version="1.0" encoding="utf-8"?>
-<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
-
-	<category dsq:id="707279">
-		<forum>radiot</forum>
-		<title>General</title>
-		<isDefault>true</isDefault>
-	</category>
-
-	<thread dsq:id="247918464">
-		<id/>
-		<forum>radiot</forum>
-		<category dsq:id="707279"/>
-		<link>http://radio-t.umputun.com/2011/03/229_8880.html</link>
-		<title>Радио-Т: Радио-Т 229</title>
-		<message/>
-		<createdAt>2011-03-07T20:46:25Z</createdAt>
-		<author>
-			<email>umputun@gmail.com</email>
-			<name>Umputun</name>
-			<isAnonymous>false</isAnonymous>
-			<username>umputun</username>
-		</author>
-		<ipAddress>98.212.28.115</ipAddress>
-		<isClosed>false</isClosed>
-		<isDeleted>false</isDeleted>
-	</thread>
-	<thread dsq:id="247937687">
-		<id>http://www.radio-t.com/p/2011/03/05/podcast-229/</id>
-		<forum>radiot</forum>
-		<category dsq:id="707279"/>
-		<link>https://radio-t.com/p/2011/03/05/podcast-229/</link>
-		<title>Радио-Т: Радио-Т 229</title>
-		<message/>
-		<createdAt>2011-03-07T21:17:17Z</createdAt>
-		<author>
-			<email>umputun@gmail.com</email>
-			<name>Umputun</name>
-			<isAnonymous>false</isAnonymous>
-			<username>umputun</username>
-		</author>
-		<ipAddress>80.250.214.235</ipAddress>
-		<isClosed>true</isClosed>
-		<isDeleted>false</isDeleted>
-	</thread>
-
-	<post dsq:id="299619020">
-		<id>3565798471341011339</id>
-		<message>
-			<![CDATA[<p>The quick brown fox jumps over the lazy dog.</p><p><a href="https://https://radio-t.com" rel="nofollow noopener" title="radio-t">some link</a></p>]]>
-		</message>
-		<createdAt>2011-08-31T15:16:29Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email/>
-			<name>Alexander Blah</name>
-			<isAnonymous>false</isAnonymous>
-			<username>facebook-1787732238</username>
-		</author>
-		<ipAddress>178.178.178.178</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-
-	<post dsq:id="299744309">
-		<id>3029154520436241933</id>
-		<message>
-			<![CDATA[<p>Microsoft показал проводник Windows 8 с ленточным интерфейсом.</p><p><a href="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx" rel="nofollow noopener" title="http://blogs.msdn.com/b/b8/archive/2011/08/29/improvements-in-windows-explorer.aspx">http://blogs.msdn.com/b/b8/...</a> </p>]]>
-		</message>
-		<createdAt>2011-08-31T17:44:22Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email>mihail.noname@gmail.com</email>
-			<name>mikhail</name>
-			<isAnonymous>false</isAnonymous>
-			<username>mikhail-noname</username>
-		</author>
-		<ipAddress>195.195.195.139</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-
-	<post dsq:id="299986072">
-		<id>6580890074280459209</id>
-		<message>
-			<![CDATA[<p>Google App Engine скоро выходит из превью статуса.</p><p>Сейчас письмо пришло от гугла.</p><p>Для платных приложений использущих High Replication Datastore (HRD) будет 99.95% uptime SLA.<br>Будут Премьер аккаунты за 500 баксов/месяц с оперативной поддержкой и любым количеством приложений на аккаунте (+ плата за потребленные ресурсы).<br>В связи с переходом на новую систему оплаты, обещают снизить бесплатные квоты.<br>Всем кто включит биллинг до 31 октября, обещают 50 баксов :)</p>]]>
-		</message>
-		<createdAt>2011-08-31T22:48:43Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>false</isSpam>
-		<author>
-			<email>dmitri.noname@gmail.com</email>
-			<name>Dmitry Noname</name>
-			<isAnonymous>false</isAnonymous>
-			<username>google-74b9e7568ef6860e93862c5d77590123</username>
-		</author>
-		<ipAddress>89.89.89.139</ipAddress>
-		<thread dsq:id="247918464"/>
-	</post>
-
-	<post dsq:id="299986073">
-		<id>6580890074280459219</id>
-		<message>some ugly spam</message>
-		<createdAt>2011-09-30T22:48:43Z</createdAt>
-		<isDeleted>false</isDeleted>
-		<isSpam>true</isSpam>
-		<author>
-			<email>spam.noname@gmail.com</email>
-			<name>Spam Noname</name>
-			<isAnonymous>false</isAnonymous>
-			<username>google-2c5d77590123</username>
-		</author>
-		<ipAddress>189.89.89.139</ipAddress>
-		<thread dsq:id="247937687"/>
-	</post>
-</disqus>
-`

app/migrator/migrator.go

@@ -1,67 +0,0 @@
-// Package migrator provides import/export functionality. It defines Importer and Exporter interfaces
-// amd implements for disqus (importer only) and "native" remark (both importer and exporter).
-// Also implements AutoBackup scheduler running exports as backups and saving them locally.
-package migrator
-
-import (
-	"io"
-	"log"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/store"
-)
-
-// Importer defines interface to convert posts from external sources
-type Importer interface {
-	Import(r io.Reader, siteID string) (int, error)
-}
-
-// Exporter defines interface to export comments from internal store
-type Exporter interface {
-	Export(w io.Writer, siteID string) (int, error)
-}
-
-// Store defines minimal interface needed to export and import comments
-type Store interface {
-	Create(comment store.Comment) (commentID string, err error)
-	Find(locator store.Locator, sort string) ([]store.Comment, error)
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)
-	DeleteAll(siteID string) error
-}
-
-// ImportParams defines everything needed to run import
-type ImportParams struct {
-	DataStore Store
-	InputFile string
-	Provider  string
-	SiteID    string
-}
-
-// ImportComments imports from given provider format and saves to store
-func ImportComments(p ImportParams) (int, error) {
-	log.Printf("[INFO] import from %s (%s) to %s", p.InputFile, p.Provider, p.SiteID)
-
-	var importer Importer
-	switch p.Provider {
-	case "disqus":
-		importer = &Disqus{DataStore: p.DataStore}
-	case "native":
-		importer = &Remark{DataStore: p.DataStore}
-	default:
-		return 0, errors.Errorf("unsupported import provider %s", p.Provider)
-	}
-
-	fh, err := os.Open(p.InputFile)
-	if err != nil {
-		return 0, errors.Wrapf(err, "can't open import file %s", p.InputFile)
-	}
-
-	defer func() {
-		if err = fh.Close(); err != nil {
-			log.Printf("[WARN] can't close %s, %s", p.InputFile, err)
-		}
-	}()
-
-	return importer.Import(fh, p.SiteID)
-}

app/migrator/migrator_test.go

@@ -1,93 +0,0 @@
-package migrator
-
-import (
-	"io/ioutil"
-	"os"
-	"testing"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store/service"
-
-	"github.com/umputun/remark/app/store/engine"
-)
-
-func TestMigrator_ImportDisqus(t *testing.T) {
-	defer func() {
-		os.Remove("/tmp/remark-test.db")
-		os.Remove("/tmp/disqus-test.xml")
-	}()
-
-	err := ioutil.WriteFile("/tmp/disqus-test.xml", []byte(xmlTest), 0600)
-	require.Nil(t, err)
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-	size, err := ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.xml",
-		SiteID:    "test",
-		Provider:  "disqus",
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 3, size)
-
-	last, err := dataStore.Last("test", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(last), "3 comments imported")
-}
-
-func TestMigrator_ImportRemark(t *testing.T) {
-	defer func() {
-		os.Remove("/tmp/remark-test.db")
-		os.Remove("/tmp/disqus-test.r42")
-	}()
-
-	data := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n" +
-		`{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	err := ioutil.WriteFile("/tmp/disqus-test.r42", []byte(data), 0600)
-	require.Nil(t, err)
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "radio-t"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-
-	size, err := ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.r42",
-		SiteID:    "radio-t",
-		Provider:  "native",
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	last, err := dataStore.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(last), "2 comments imported")
-}
-
-func TestMigrator_ImportFailed(t *testing.T) {
-
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/remark-test.db", SiteID: "test"})
-	require.Nil(t, err, "create store")
-	dataStore := &service.DataStore{Interface: b}
-	_, err = ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test.xml",
-		SiteID:    "test",
-		Provider:  "bad",
-	})
-
-	assert.EqualError(t, err, "unsupported import provider bad")
-
-	_, err = ImportComments(ImportParams{
-		DataStore: dataStore,
-		InputFile: "/tmp/disqus-test-bad.xml",
-		SiteID:    "test",
-		Provider:  "native",
-	})
-	assert.EqualError(t, err, "can't open import file /tmp/disqus-test-bad.xml: open /tmp/disqus-test-bad.xml: no such file or directory")
-}

app/migrator/remark.go

@@ -1,95 +0,0 @@
-package migrator
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"io"
-	"log"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/store"
-)
-
-// Remark implements exporter and importer for internal store format
-type Remark struct {
-	DataStore Store
-}
-
-// Export all comments to writer as json strings. Each comment is one string, separated by "\n"
-func (r *Remark) Export(w io.Writer, siteID string) (size int, err error) {
-	topics, err := r.DataStore.List(siteID, 0, 0)
-	if err != nil {
-		return 0, err
-	}
-	log.Printf("[DEBUG] exporting %d topics", len(topics))
-
-	commentsCount := 0
-	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
-		topic := topics[i]
-		comments, err := r.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
-			return commentsCount, err
-		}
-
-		for _, comment := range comments {
-
-			buf := &bytes.Buffer{}
-			enc := json.NewEncoder(buf)
-			enc.SetEscapeHTML(false)
-
-			if err := enc.Encode(comment); err != nil {
-				return commentsCount, errors.Wrapf(err, "can't marshal %v", comments)
-			}
-			data := buf.Bytes()
-			if _, err := w.Write(data); err != nil {
-				return commentsCount, errors.Wrap(err, "can't write comment data")
-			}
-			commentsCount++
-		}
-	}
-	log.Printf("[DEBUG] exported %d comments", commentsCount)
-	return commentsCount, nil
-}
-
-// Import comments from json strings produced by Remark.Export
-func (r *Remark) Import(reader io.Reader, siteID string) (size int, err error) {
-
-	if err := r.DataStore.DeleteAll(siteID); err != nil {
-		return 0, err
-	}
-
-	failed := 0
-	total, comments := 0, 0
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		rec := scanner.Bytes()
-		if len(rec) < 3 {
-			continue
-		}
-		total++
-		comment := store.Comment{}
-		if err := json.Unmarshal(rec, &comment); err != nil {
-			failed++
-			log.Printf("[WARN] unmarshal failed for %s, %s", string(rec), err)
-			continue
-		}
-		if _, err := r.DataStore.Create(comment); err != nil {
-			failed++
-			log.Printf("[WARN] can't write %+v to store, %s", comment, err)
-			continue
-		}
-		comments++
-		if comments%1000 == 0 {
-			log.Printf("[DEBUG] imported %d comments", comments)
-		}
-	}
-	if scanner.Err() != nil {
-		return comments, errors.Wrap(scanner.Err(), "error in scan")
-	}
-	if failed > 0 {
-		return comments, errors.Errorf("failed to save %d comments", failed)
-	}
-	log.Printf("[INFO] imported %d comments from %d records", comments, total)
-	return comments, nil
-}

app/migrator/remark_test.go

@@ -1,112 +0,0 @@
-package migrator
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestRemark_Export(t *testing.T) {
-	b := prep(t)
-	r := Remark{DataStore: b}
-
-	buf := &bytes.Buffer{}
-	size, err := r.Export(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	c1, err := buf.ReadString('\n')
-	assert.Nil(t, err)
-	log.Print(c1)
-	exp := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-	assert.Equal(t, exp, c1)
-}
-
-func TestRemark_Import(t *testing.T) {
-	r1 := `{"id":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	r2 := `{"id":"afbc17f177ee1a1c0ee6e1e025749966ec071adc","pid":"efbc17f177ee1a1c0ee6e1e025749966ec071adc","text":"some text2, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:23-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	buf.WriteString(r1)
-	buf.WriteString(r2)
-	buf.WriteString("{}")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	size, err := r.Import(buf, "radio-t")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, size)
-
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments))
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[0].ID)
-	assert.Equal(t, "afbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ID)
-	assert.Equal(t, "efbc17f177ee1a1c0ee6e1e025749966ec071adc", comments[1].ParentID)
-}
-
-func TestRemark_ImportManyWithError(t *testing.T) {
-	goodRec := `{"id":"%d","pid":"","text":"some text, <a href=\"http://radio-t.com\" rel=\"nofollow\">link</a>","user":{"name":"user name","id":"user1","picture":"","profile":"","admin":false},"locator":{"site":"radio-t","url":"https://radio-t.com"},"score":0,"votes":{},"time":"2017-12-20T15:18:22-06:00"}` + "\n"
-
-	buf := &bytes.Buffer{}
-	for i := 0; i < 1200; i++ {
-		buf.WriteString(fmt.Sprintf(goodRec, i))
-	}
-	buf.WriteString("bad1\n")
-	buf.WriteString("bad2\n")
-
-	os.Remove(testDb)
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-	r := Remark{DataStore: &service.DataStore{Interface: b}}
-	n, err := r.Import(buf, "radio-t")
-	assert.EqualError(t, err, "failed to save 2 comments")
-	assert.Equal(t, 1200, n)
-	comments, err := b.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1200, len(comments))
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *service.DataStore {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{SiteID: "radio-t", FileName: testDb})
-	assert.Nil(t, err)
-
-	b := &service.DataStore{Interface: boltStore}
-
-	comment := store.Comment{
-		ID:        "efbc17f177ee1a1c0ee6e1e025749966ec071adc",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		Text: "some text2", Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/rest/api/admin.go

@@ -1,245 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	"github.com/umputun/remark/app/rest/auth"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// admin provides router for all requests available for admin users only
-type admin struct {
-	dataService   *service.DataStore
-	exporter      migrator.Exporter
-	cache         cache.LoadingCache
-	authenticator auth.Authenticator
-	readOnlyAge   int
-}
-
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Get("/export", a.exportCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	return router
-}
-
-// DELETE /comment/{id}?site=siteID&url=post-url - removes comment
-func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	id := chi.URLParam(r, "id")
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	log.Printf("[INFO] delete comment %s", id)
-
-	err := a.dataService.Delete(locator, id, store.SoftDelete)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
-		return
-	}
-	a.cache.Flush(locator.SiteID, locator.URL)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"id": id, "locator": locator})
-}
-
-// DELETE /user/{userid}?site=side-id - delete all user comments for requested userid
-func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
-
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[INFO] delete all user comments for %s, site %s", userID, siteID)
-
-	if err := a.dataService.DeleteUser(siteID, userID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
-}
-
-// GET /deleteme?token=jwt - delete all user comments by user's request. Gets info about deleted used from provided token
-// request made GET to allow direct click from the email sent by user
-func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
-
-	token := r.URL.Query().Get("token")
-
-	claims, err := a.authenticator.JWTService.Parse(token)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't process token")
-		return
-	}
-
-	log.Printf("[INFO] delete all user comments by request for %s, site %s", claims.User.ID, claims.SiteID)
-
-	if err := a.dataService.DeleteUser(claims.SiteID, claims.User.ID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
-		return
-	}
-	a.cache.Flush(claims.SiteID, claims.User.ID)
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
-}
-
-// PUT /user/{userid}?site=side-id&block=1 - block or unblock user
-func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	blockStatus := r.URL.Query().Get("block") == "1"
-
-	if err := a.dataService.SetBlock(siteID, userID, blockStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
-}
-
-// GET /blocked?site=siteID - list blocked users
-func (a *admin) blockedUsersCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	users, err := a.dataService.Blocked(siteID)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get blocked users")
-		return
-	}
-	render.JSON(w, r, users)
-}
-
-// PUT /readonly?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
-func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	roStatus := r.URL.Query().Get("ro") == "1"
-
-	isRoByAge := func(info store.PostInfo) bool {
-		return a.readOnlyAge > 0 && !info.FirstTS.IsZero() &&
-			info.FirstTS.AddDate(0, 0, a.readOnlyAge).Before(time.Now())
-	}
-
-	// don't allow to reset ro for posts turned to ro by ReadOnlyAge
-	if !roStatus {
-		if info, e := a.dataService.Info(locator, a.readOnlyAge); e == nil && isRoByAge(info) {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "read-only due the age")
-			return
-		}
-	}
-
-	if err := a.dataService.SetReadOnly(locator, roStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
-		return
-	}
-	a.cache.Flush(locator.SiteID)
-	render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus})
-}
-
-// PUT /verify?site=siteID&url=post-url&ro=1 - set or reset read-only status for the post
-func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
-	userID := chi.URLParam(r, "userid")
-	siteID := r.URL.Query().Get("site")
-	verifyStatus := r.URL.Query().Get("verified") == "1"
-
-	if err := a.dataService.SetVerified(siteID, userID, verifyStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
-		return
-	}
-	a.cache.Flush(siteID, userID)
-	render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus})
-}
-
-// PUT /pin/{id}?site=siteID&url=post-url&pin=1
-// mark/unmark comment as a special
-func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
-	commentID := chi.URLParam(r, "id")
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	pinStatus := r.URL.Query().Get("pin") == "1"
-
-	if err := a.dataService.SetPin(locator, commentID, pinStatus); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
-		return
-	}
-	a.cache.Flush(locator.URL)
-	render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus})
-}
-
-// GET /export?site=site-id?mode=file|stream
-// exports all comments for siteID as json stream or gz file
-func (a *admin) exportCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	var writer io.Writer = w
-	if r.URL.Query().Get("mode") == "file" {
-		exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-		w.Header().Set("Content-Type", "application/gzip")
-		w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-		w.WriteHeader(http.StatusOK)
-		gzWriter := gzip.NewWriter(w)
-		defer func() {
-			if e := gzWriter.Close(); e != nil {
-				log.Printf("[WARN] can't close gzip writer, %s", e)
-			}
-		}()
-		writer = gzWriter
-	}
-
-	if _, err := a.exporter.Export(writer, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
-}
-
-func (a *admin) checkBlocked(siteID string, user store.User) bool {
-	return a.dataService.IsBlocked(siteID, user.ID)
-}
-
-// post-processes comments, hides text of all comments for blocked users,
-// resets score and votes too. Also hides sensitive info for non-admin users
-func (a *admin) alterComments(comments []store.Comment, r *http.Request) (res []store.Comment) {
-	res = make([]store.Comment, len(comments))
-
-	user, err := rest.GetUserInfo(r)
-	isAdmin := err == nil && user.Admin // make separate cache key for admins
-
-	for i, c := range comments {
-
-		blocked := a.dataService.IsBlocked(c.Locator.SiteID, c.User.ID)
-		// process blocked users
-		if blocked {
-			if !isAdmin { // reset comment to deleted for non-admins
-				c.SetDeleted(store.SoftDelete)
-			}
-			c.User.Blocked = true
-			c.Deleted = true
-		}
-
-		// set verified status retroactively
-		if !blocked {
-			c.User.Verified = a.dataService.IsVerified(c.Locator.SiteID, c.User.ID)
-		}
-
-		// hide info from non-admins
-		if !isAdmin {
-			c.User.IP = ""
-		}
-
-		res[i] = c
-	}
-	return res
-}

app/rest/api/admin_test.go

@@ -1,541 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestAdmin_Delete(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", User: store.User{ID: "id", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", User: store.User{ID: "id", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodDelete,
-		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	body, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, "", cr.Text)
-	assert.True(t, cr.Deleted)
-}
-
-func TestAdmin_DeleteUser(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Orig: "o test test #1", User: store.User{ID: "id1", Name: "name"},
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", Orig: "o test test #2", User: store.User{ID: "id2", Name: "name"}, ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c3 := store.Comment{Text: "test test #3", Orig: "o test test #3", User: store.User{ID: "id2", Name: "name"}, ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	// write comments directly to store to keep user id
-	id1, err := srv.DataService.Create(c1)
-	assert.NoError(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.NoError(t, err)
-	_, err = srv.DataService.Create(c3)
-	assert.NoError(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	// all 3 comments here, but for id2 they deleted
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
-
-	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
-
-	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
-
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
-}
-
-func TestAdmin_Pin(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	pin := func(val int) int {
-		client := http.Client{}
-		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
-		resp, err := client.Do(req)
-		assert.Nil(t, err)
-		return resp.StatusCode
-	}
-
-	code := pin(1)
-	assert.Equal(t, 200, code)
-
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.True(t, cr.Pin)
-
-	code = pin(-1)
-	assert.Equal(t, 200, code)
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr = store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.False(t, cr.Pin)
-}
-
-func TestAdmin_Block(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	block := func(val int) (code int, body []byte) {
-		client := http.Client{}
-		req, e := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val), nil)
-		assert.Nil(t, e)
-		req.SetBasicAuth("dev", "password")
-		resp, e := client.Do(req)
-		require.Nil(t, e)
-		body, e = ioutil.ReadAll(resp.Body)
-		assert.Nil(t, e)
-		resp.Body.Close()
-		return resp.StatusCode, body
-	}
-
-	code, body := block(1)
-	require.Equal(t, 200, code)
-	j := JSON{}
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, "user1", j["user_id"])
-	assert.Equal(t, true, j["block"])
-	assert.Equal(t, "radio-t", j["site_id"])
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "", comments.Comments[0].Text)
-	assert.True(t, comments.Comments[0].Deleted)
-
-	code, body = block(-1)
-	require.Equal(t, 200, code)
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, false, j["block"])
-}
-
-func TestAdmin_BlockedList(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	client := http.Client{}
-
-	// block user1
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	// block user2
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user2", 1), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	res, code := getWithAuth(t, ts.URL+"/api/v1/admin/blocked?site=radio-t")
-	require.Equal(t, 200, code, res)
-	users := []store.BlockedUser{}
-	err = json.Unmarshal([]byte(res), &users)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(users), "two users blocked")
-	assert.Equal(t, "user1", users[0].ID)
-	assert.Equal(t, "user2", users[1].ID)
-}
-
-func TestAdmin_ReadOnly(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.False(t, info.ReadOnly)
-
-	client := http.Client{}
-
-	// set post to read-only
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-	// reset post's read-only
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.False(t, info.ReadOnly)
-}
-
-func TestAdmin_ReadOnlyWithAge(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"},
-		Timestamp: time.Date(2001, 1, 1, 1, 1, 1, 0, time.Local)}
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-
-	info, err := srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 10)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly, "ro by age")
-
-	client := http.Client{}
-
-	// set post to read-only
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-	// reset post's read-only
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
-	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
-	assert.True(t, info.ReadOnly)
-
-}
-func TestAdmin_Verify(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	verified := srv.DataService.IsVerified("radio-t", "user1")
-	assert.False(t, verified)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
-	assert.True(t, verified)
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
-	assert.True(t, comments.Comments[0].User.Verified)
-
-	req, err = http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-	verified = srv.DataService.IsVerified("radio-t", "user1")
-	assert.False(t, verified)
-
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
-	assert.Equal(t, 200, code)
-	comments = commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, "test test #1", comments.Comments[0].Text)
-	assert.False(t, comments.Comments[0].User.Verified)
-
-}
-
-func TestAdmin_ExportStream(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	body, code := getWithAuth(t, ts.URL+"/api/v1/admin/export?site=radio-t&mode=stream")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, 2, strings.Count(body, "\n"))
-	assert.Equal(t, 2, strings.Count(body, "\"text\""))
-	t.Logf("%s", body)
-}
-
-func TestAdmin_ExportFile(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
-	t.Logf("%s", string(ungzBody))
-}
-
-func TestAdmin_DeleteMeRequest(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	comments, err := srv.DataService.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "a comment for user1")
-
-	claims := auth.CustomClaims{
-		SiteID:      "radio-t",
-		SessionOnly: true,
-		StandardClaims: jwt.StandardClaims{
-			Id:        "1234567",
-			Issuer:    "remark42",
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-		},
-		User: &store.User{
-			ID: "user1",
-		},
-	}
-
-	token, err := srv.Authenticator.JWTService.Token(&claims)
-	assert.Nil(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	_, err = srv.DataService.User("radio-t", "user1", 0, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-}
-
-func TestAdmin_DeleteMeRequestFailed(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user1 name", ID: "user1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah"}, User: store.User{Name: "user2", ID: "user2"}}
-
-	_, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-	_, err = srv.DataService.Create(c2)
-	assert.Nil(t, err)
-
-	// try with bad token
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, "bad token"), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
-
-	// try with bad auth
-	claims := auth.CustomClaims{
-		SiteID:      "radio-t",
-		SessionOnly: true,
-		StandardClaims: jwt.StandardClaims{
-			Id:        "1234567",
-			Issuer:    "remark42",
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-		},
-		User: &store.User{
-			ID: "user1",
-		},
-	}
-
-	token, err := srv.Authenticator.JWTService.Token(&claims)
-	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "bad-password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-
-	// try bad user
-	badClaims := claims
-	badClaims.User.ID = "no-such-id"
-	token, err = srv.Authenticator.JWTService.Token(&badClaims)
-	assert.Nil(t, err)
-	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, token), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode, resp.Status)
-
-}

app/rest/api/middleware.go

@@ -1,169 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"regexp"
-	"runtime/debug"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi/middleware"
-
-	"github.com/umputun/remark/app/rest"
-)
-
-// JSON is a map alias, just for convenience
-type JSON map[string]interface{}
-
-// AppInfo adds custom app-info to the response header
-func AppInfo(app string, version string) func(http.Handler) http.Handler {
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Org", "Umputun")
-			w.Header().Set("App-Name", app)
-			w.Header().Set("App-Version", version)
-			if mhost := os.Getenv("MHOST"); mhost != "" {
-				w.Header().Set("Host", mhost)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// Ping middleware response with pong to /ping. Stops chain if ping request detected
-func Ping(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		if r.Method == "GET" && strings.HasSuffix(strings.ToLower(r.URL.Path), "/ping") {
-			w.Header().Set("Content-Type", "text/plain")
-			w.WriteHeader(http.StatusOK)
-			if _, err := w.Write([]byte("pong")); err != nil {
-				log.Printf("[WARN] can't send pong, %s", err)
-			}
-			return
-		}
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// Recoverer is a middleware that recovers from panics, logs the panic and returns a HTTP 500 status if possible.
-func Recoverer(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		defer func() {
-			if rvr := recover(); rvr != nil {
-				log.Printf("[WARN] request panic, %v", rvr)
-				debug.PrintStack()
-				http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			}
-		}()
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-// LoggerFlag type
-type LoggerFlag int
-
-// logger flags enum
-const (
-	LogAll LoggerFlag = iota
-	LogUser
-	LogBody
-	LogNone
-)
-const maxBody = 1024
-
-var reMultWhtsp = regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-// Logger middleware prints http log. Customized by set of LoggerFlag
-func Logger(ipFn func(ip string) string, flags ...LoggerFlag) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if inLogFlags(LogNone, flags) { // skip logging
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			ww := middleware.NewWrapResponseWriter(w, 1)
-			body, user := getBodyAndUser(r, flags)
-			t1 := time.Now()
-			defer func() {
-				t2 := time.Now()
-
-				q := r.URL.String()
-				if qun, err := url.QueryUnescape(q); err == nil {
-					q = qun
-				}
-
-				remoteIP := strings.Split(r.RemoteAddr, ":")[0]
-				if strings.HasPrefix(r.RemoteAddr, "[") {
-					remoteIP = strings.Split(r.RemoteAddr, "]:")[0] + "]"
-				}
-				if ipFn != nil {
-					remoteIP = ipFn(remoteIP)
-				}
-
-				log.Printf("[INFO] REST %s - %s - %s - %d (%d) - %v %s %s",
-					r.Method, q, remoteIP, ww.Status(), ww.BytesWritten(), t2.Sub(t1), user, body)
-			}()
-
-			h.ServeHTTP(ww, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-
-	return f
-}
-
-func getBodyAndUser(r *http.Request, flags []LoggerFlag) (body string, user string) {
-	ctx := r.Context()
-	if ctx == nil {
-		return "", ""
-	}
-
-	if inLogFlags(LogBody, flags) {
-		if content, err := ioutil.ReadAll(r.Body); err == nil {
-			body = string(content)
-			r.Body = ioutil.NopCloser(bytes.NewReader(content))
-
-			if len(body) > 0 {
-				body = strings.Replace(body, "\n", " ", -1)
-				body = reMultWhtsp.ReplaceAllString(body, " ")
-			}
-
-			if len(body) > maxBody {
-				body = body[:maxBody] + "..."
-			}
-		}
-	}
-
-	if inLogFlags(LogUser, flags) {
-		u, err := rest.GetUserInfo(r)
-		if err == nil && u.Name != "" {
-			user = fmt.Sprintf(" - %s %q", u.ID, u.Name)
-		}
-	}
-
-	return body, user
-}
-
-func inLogFlags(f LoggerFlag, flags []LoggerFlag) bool {
-	for _, flg := range flags {
-		if (flg == LogAll && f != LogNone) || flg == f {
-			return true
-		}
-	}
-	return false
-}

app/rest/api/middleware_test.go

@@ -1,63 +0,0 @@
-package api
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestMiddleware_AppInfo(t *testing.T) {
-	router := chi.NewRouter()
-	router.With(AppInfo("remark42", "12345")).Get("/blah", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
-		w.Write([]byte("blah blah"))
-	})
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/blah")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "blah blah", string(b))
-	assert.Equal(t, "remark42", resp.Header.Get("App-Name"))
-	assert.Equal(t, "12345", resp.Header.Get("App-Version"))
-	assert.Equal(t, "Umputun", resp.Header.Get("Org"))
-}
-
-func TestMiddleware_GetBodyAndUser(t *testing.T) {
-	req, err := http.NewRequest("GET", "http://example.com/request", strings.NewReader("body"))
-	require.Nil(t, err)
-
-	body, user := getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, "body", body)
-	assert.Equal(t, "", user, "no user")
-
-	req = rest.SetUserInfo(req, store.User{ID: "id1", Name: "user1"})
-	_, user = getBodyAndUser(req, []LoggerFlag{LogAll})
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-
-	body, user = getBodyAndUser(req, nil)
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogNone})
-	assert.Equal(t, "", body)
-	assert.Equal(t, "", user, "no user")
-
-	body, user = getBodyAndUser(req, []LoggerFlag{LogUser})
-	assert.Equal(t, "", body)
-	assert.Equal(t, ` - id1 "user1"`, user, "no user")
-}

app/rest/api/migrator.go

@@ -1,133 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"context"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-)
-
-// Migrator rest runs on unexposed port and available for local requests only
-type Migrator struct {
-	Version        string
-	Cache          cache.LoadingCache
-	NativeImporter migrator.Importer
-	DisqusImporter migrator.Importer
-	NativeExported migrator.Exporter
-	SecretKey      string
-
-	httpServer *http.Server
-	lock       sync.Mutex
-}
-
-// Run the listener and request's router, activate rest server
-// this server doesn't have any authentication and SHOULDN'T BE EXPOSED in any way
-func (m *Migrator) Run(port int) {
-	log.Printf("[INFO] activate import server on port %d", port)
-	router := m.routes()
-
-	m.lock.Lock()
-	m.httpServer = &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: router}
-	m.lock.Unlock()
-
-	err := m.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown import http server
-func (m *Migrator) Shutdown() {
-	log.Print("[WARN] shutdown import server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-
-	m.lock.Lock()
-	if err := m.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] importer shutdown error, %s", err)
-	}
-	m.lock.Unlock()
-
-	log.Print("[DEBUG] shutdown import server completed")
-}
-
-func (m *Migrator) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(15*time.Minute))
-	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-	router.Use(AppInfo("remark42-migrator", m.Version), Ping, Logger(nil, LogAll))
-	router.Post("/api/v1/admin/import", m.importCtrl)
-	router.Get("/api/v1/admin/export", m.exportCtrl)
-	return router
-}
-
-// POST /import?secret=key&site=site-id&provider=disqus|remark
-// imports comments from post body.
-func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-	importer := m.NativeImporter
-	if r.URL.Query().Get("provider") == "disqus" {
-		importer = m.DisqusImporter
-	}
-
-	size, err := importer.Import(r.Body, siteID)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
-		return
-	}
-	m.Cache.Flush(siteID)
-
-	render.Status(r, http.StatusCreated)
-	render.JSON(w, r, JSON{"status": "ok", "size": size})
-}
-
-// GET /export?site=site-id&secret=12345
-// exports all comments for siteID as gz file
-func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
-
-	secret := r.URL.Query().Get("secret")
-	if strings.TrimSpace(secret) == "" || secret != m.SecretKey {
-		render.Status(r, http.StatusForbidden)
-		render.JSON(w, r, JSON{"status": "error", "details": "secret key"})
-		return
-	}
-
-	siteID := r.URL.Query().Get("site")
-
-	exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-	w.WriteHeader(http.StatusOK)
-	gzWriter := gzip.NewWriter(w)
-	defer func() {
-		if e := gzWriter.Close(); e != nil {
-			log.Printf("[WARN] can't close gzip writer, %s", e)
-		}
-	}()
-
-	if _, err := m.NativeExported.Export(gzWriter, siteID); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "export failed")
-		return
-	}
-}

app/rest/api/migrator_test.go

@@ -1,126 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-func TestMigrator_Import(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, `{"size":2,"status":"ok"}`+"\n", string(b))
-}
-
-func TestMigrator_ImportRejected(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
-	assert.Nil(t, err)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-}
-
-func TestMigrator_Export(t *testing.T) {
-	srv, ts := prepImportSrv(t)
-	assert.NotNil(t, srv)
-	defer cleanupImportSrv(srv, ts)
-
-	r := strings.NewReader(`{"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
-	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=123456", r)
-	require.Nil(t, err)
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=123456", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\n"))
-	assert.Equal(t, 2, strings.Count(string(ungzBody), "\"text\""))
-	t.Logf("%s", string(ungzBody))
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&secret=bad", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 403, resp.StatusCode)
-}
-
-func TestMigrator_Shutdown(t *testing.T) {
-	srv := Migrator{}
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b}
-	svc = &Migrator{
-		DisqusImporter: &migrator.Disqus{DataStore: dataStore},
-		NativeImporter: &migrator.Remark{DataStore: dataStore},
-		NativeExported: &migrator.Remark{DataStore: dataStore},
-		Cache:          &mockCache{},
-		SecretKey:      "123456",
-	}
-
-	routes := svc.routes()
-	ts = httptest.NewServer(routes)
-	return svc, ts
-}
-
-func cleanupImportSrv(srv *Migrator, ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-}

app/rest/api/rest.go

@@ -1,249 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/didip/tollbooth"
-	"github.com/didip/tollbooth_chi"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
-	"github.com/go-chi/render"
-	"github.com/pkg/errors"
-	"gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// Rest is a rest access server
-type Rest struct {
-	Version         string
-	DataService     *service.DataStore
-	Authenticator   auth.Authenticator
-	Exporter        migrator.Exporter
-	Cache           cache.LoadingCache
-	AvatarProxy     *proxy.Avatar
-	ImageProxy      *proxy.Image
-	WebRoot         string
-	RemarkURL       string
-	ReadOnlyAge     int
-	ScoreThresholds struct {
-		Low      int
-		Critical int
-	}
-
-	httpServer *http.Server
-	lock       sync.Mutex
-
-	adminService admin
-}
-
-const hardBodyLimit = 1024 * 64 // limit size of body
-
-var mdExt = blackfriday.NoIntraEmphasis | blackfriday.Tables | blackfriday.FencedCode |
-	blackfriday.Strikethrough | blackfriday.SpaceHeadings | blackfriday.HardLineBreak |
-	blackfriday.BackslashLineBreak | blackfriday.Autolink
-
-type commentsWithInfo struct {
-	Comments []store.Comment `json:"comments"`
-	Info     store.PostInfo  `json:"info,omitempty"`
-}
-
-// Run the lister and request's router, activate rest server
-func (s *Rest) Run(port int) {
-	log.Printf("[INFO] activate rest server on port %d", port)
-
-	if len(s.Authenticator.Admins) > 0 {
-		log.Printf("[DEBUG] admins %+v", s.Authenticator.Admins)
-	}
-
-	router := s.routes()
-
-	s.lock.Lock()
-	s.httpServer = &http.Server{
-		Addr:              fmt.Sprintf(":%d", port),
-		Handler:           router,
-		ReadHeaderTimeout: 5 * time.Second,
-		WriteTimeout:      5 * time.Second,
-		IdleTimeout:       30 * time.Second,
-	}
-	s.lock.Unlock()
-
-	err := s.httpServer.ListenAndServe()
-	log.Printf("[WARN] http server terminated, %s", err)
-}
-
-// Shutdown rest http server
-func (s *Rest) Shutdown() {
-	log.Print("[WARN] shutdown rest server")
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-	defer cancel()
-	s.lock.Lock()
-	if err := s.httpServer.Shutdown(ctx); err != nil {
-		log.Printf("[DEBUG] rest shutdown error, %s", err)
-	}
-	log.Print("[DEBUG] shutdown rest server completed")
-	s.lock.Unlock()
-}
-
-func (s *Rest) routes() chi.Router {
-	router := chi.NewRouter()
-	router.Use(middleware.RealIP, Recoverer)
-	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
-	router.Use(AppInfo("remark42", s.Version), Ping)
-
-	s.adminService = admin{
-		dataService:   s.DataService,
-		exporter:      s.Exporter,
-		cache:         s.Cache,
-		authenticator: s.Authenticator,
-		readOnlyAge:   s.ReadOnlyAge,
-	}
-
-	ipFn := func(ip string) string { return store.HashValue(ip, s.DataService.Secret)[:12] } // logger uses it for anonymization
-
-	// auth routes for all providers
-	router.Route("/auth", func(r chi.Router) {
-		r.Use(Logger(ipFn, LogAll), tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
-		for _, provider := range s.Authenticator.Providers {
-			r.Mount("/"+provider.Name, provider.Routes()) // mount auth providers as /auth/{name}
-		}
-		if len(s.Authenticator.Providers) > 0 {
-			// shortcut, can be any of providers, all logouts do the same - removes cookie
-			r.Get("/logout", s.Authenticator.Providers[0].LogoutHandler)
-		}
-	})
-
-	avatarMiddlewares := []func(http.Handler) http.Handler{
-		Logger(ipFn, LogNone),
-		tollbooth_chi.LimitHandler(tollbooth.NewLimiter(100, nil)),
-	}
-	router.Mount(s.AvatarProxy.Routes(avatarMiddlewares...)) // mount avatars to /api/v1/avatar/{file.img}
-
-	// api routes
-	router.Route("/api/v1", func(rapi chi.Router) {
-		rapi.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-
-		// open routes
-		rapi.Group(func(ropen chi.Router) {
-			ropen.Use(s.Authenticator.Auth(false))
-			ropen.Use(Logger(ipFn, LogAll))
-			ropen.Get("/find", s.findCommentsCtrl)
-			ropen.Get("/id/{id}", s.commentByIDCtrl)
-			ropen.Get("/comments", s.findUserCommentsCtrl)
-			ropen.Get("/last/{limit}", s.lastCommentsCtrl)
-			ropen.Get("/count", s.countCtrl)
-			ropen.Post("/counts", s.countMultiCtrl)
-			ropen.Get("/list", s.listCtrl)
-			ropen.Get("/config", s.configCtrl)
-			ropen.Post("/preview", s.previewCommentCtrl)
-			ropen.Get("/info", s.infoCtrl)
-
-			ropen.Mount("/rss", s.rssRoutes())
-			ropen.Mount("/img", s.ImageProxy.Routes())
-		})
-
-		// protected routes, require auth
-		rapi.Group(func(rauth chi.Router) {
-			rauth.Use(s.Authenticator.Auth(true))
-			rauth.Use(Logger(ipFn, LogAll))
-			rauth.Post("/comment", s.createCommentCtrl)
-			rauth.Put("/comment/{id}", s.updateCommentCtrl)
-			rauth.Get("/user", s.userInfoCtrl)
-			rauth.Put("/vote/{id}", s.voteCtrl)
-			rauth.Get("/userdata", s.userAllDataCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
-
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(s.Authenticator.AdminOnly, Logger(nil, LogAll)))
-		})
-	})
-
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// file server for static content from /web
-	addFileServer(router, "/web", http.Dir(s.WebRoot))
-	return router
-}
-
-// serves static files from /web
-func addFileServer(r chi.Router, path string, root http.FileSystem) {
-	log.Printf("[INFO] run file server for %s, path %s", root, path)
-	origPath := path
-	fs := http.StripPrefix(path, http.FileServer(root))
-	if path != "/" && path[len(path)-1] != '/' {
-		r.Get(path, http.RedirectHandler(path+"/", 301).ServeHTTP)
-		path += "/"
-	}
-	path += "*"
-
-	r.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(20, nil))).
-		Get(path, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// don't show dirs, just serve files
-			if strings.HasSuffix(r.URL.Path, "/") && len(r.URL.Path) > 1 && r.URL.Path != (origPath+"/") {
-				http.NotFound(w, r)
-				return
-			}
-			fs.ServeHTTP(w, r)
-		}))
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONWithHTML(w http.ResponseWriter, r *http.Request, v interface{}) {
-	data, err := encodeJSONWithHTML(v)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't render json response")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-func encodeJSONWithHTML(v interface{}) ([]byte, error) {
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false)
-	if err := enc.Encode(v); err != nil {
-		return nil, errors.Wrap(err, "json encoding failed")
-	}
-	return buf.Bytes(), nil
-}
-
-// renderJSONWithHTML allows html tags and forces charset=utf-8
-func renderJSONFromBytes(w http.ResponseWriter, r *http.Request, data []byte) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	if status, ok := r.Context().Value(render.StatusCtxKey).(int); ok {
-		w.WriteHeader(status)
-	}
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-func filterComments(comments []store.Comment, fn func(c store.Comment) bool) (filtered []store.Comment) {
-	for _, c := range comments {
-		if fn(c) {
-			filtered = append(filtered, c)
-		}
-	}
-	return filtered
-}

app/rest/api/rest_private.go

@@ -1,262 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"net/http"
-	"strings"
-	"time"
-
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	multierror "github.com/hashicorp/go-multierror"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/service"
-)
-
-// POST /comment - adds comment, resets all immutable fields
-func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	comment := store.Comment{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	log.Printf("[DEBUG] create comment %+v", comment)
-
-	comment.PrepareUntrusted() // clean all fields user not supposed to set
-	comment.User = user
-	comment.User.IP = strings.Split(r.RemoteAddr, ":")[0]
-
-	comment.Orig = comment.Text // original comment text, prior to md render
-	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
-		return
-	}
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
-	comment.Text = s.ImageProxy.Convert(comment.Text)
-	// check if user blocked
-	if s.adminService.checkBlocked(comment.Locator.SiteID, comment.User) {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "user blocked")
-		return
-	}
-
-	if s.ReadOnlyAge > 0 {
-		if info, e := s.DataService.Info(comment.Locator, s.ReadOnlyAge); e == nil && info.ReadOnly {
-			rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "old post, read-only")
-			return
-		}
-	}
-
-	id, err := s.DataService.Create(comment)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't save comment")
-		return
-	}
-
-	// DataService modifies comment
-	finalComment, err := s.DataService.Get(comment.Locator, id)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
-		return
-	}
-	s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)
-
-	render.Status(r, http.StatusCreated)
-	render.JSON(w, r, &finalComment)
-}
-
-// PUT /comment/{id}?site=siteID&url=post-url - update comment
-func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
-
-	edit := struct {
-		Text    string
-		Summary string
-	}{}
-
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &edit); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	id := chi.URLParam(r, "id")
-
-	log.Printf("[DEBUG] update comment %s", id)
-
-	var currComment store.Comment
-	if currComment, err = s.DataService.Get(locator, id); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comment")
-		return
-	}
-
-	if currComment.User.ID != user.ID {
-		rest.SendErrorJSON(w, r, http.StatusForbidden, errors.New("rejected"), "can not edit comments for other users")
-		return
-	}
-
-	text := string(blackfriday.Run([]byte(edit.Text), blackfriday.WithExtensions(mdExt))) // render markdown
-	text = s.ImageProxy.Convert(text)
-	editReq := service.EditRequest{
-		Text:    text,
-		Orig:    edit.Text,
-		Summary: edit.Summary,
-	}
-
-	res, err := s.DataService.EditComment(locator, id, editReq)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment")
-		return
-	}
-
-	s.Cache.Flush(locator.URL, "last", user.ID)
-	render.JSON(w, r, res)
-}
-
-// GET /user?site=siteID - returns user info
-func (s *Rest) userInfoCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-
-	if siteID := r.URL.Query().Get("site"); siteID != "" {
-		user.Verified = s.DataService.IsVerified(siteID, user.ID)
-	}
-
-	render.JSON(w, r, user)
-}
-
-// PUT /vote/{id}?site=siteID&url=post-url&vote=1 - vote for/against comment
-func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	id := chi.URLParam(r, "id")
-	log.Printf("[DEBUG] vote for comment %s", id)
-
-	vote := r.URL.Query().Get("vote") == "1"
-
-	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
-		return
-	}
-	s.Cache.Flush(locator.URL)
-	render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
-}
-
-// GET /userdata?site=siteID - exports all data about the user as a json with user info and list of all comments
-func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	userB, err := json.Marshal(&user)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user info")
-		return
-	}
-
-	exportFile := fmt.Sprintf("%s-%s-%s.json.gz", siteID, user.ID, time.Now().Format("20060102"))
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
-	gzWriter := gzip.NewWriter(w)
-	defer func() {
-		if e := gzWriter.Close(); e != nil {
-			log.Printf("[WARN] can't close gzip writer, %s", e)
-		}
-	}()
-
-	write := func(val []byte) error {
-		_, e := gzWriter.Write(val)
-		return e
-	}
-
-	var merr error
-
-	merr = multierror.Append(merr, write([]byte(`{"info": `)))     // send user prefix
-	merr = multierror.Append(merr, write(userB))                   // send user info
-	merr = multierror.Append(merr, write([]byte(`, "comments":`))) // send comments prefix
-
-	// get comments in 100 in each paginated request
-	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments")
-			return
-		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments")
-			return
-		}
-
-		merr = multierror.Append(merr, write(b))
-		if len(comments) != 100 {
-			break
-		}
-	}
-
-	merr = multierror.Append(merr, write([]byte(`}`)))
-	if merr.(*multierror.Error).ErrorOrNil() != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, merr, "can't write user info")
-		return
-	}
-
-}
-
-// POST /deleteme?site_id=site - requesting delete of all user info
-// makes jwt with user info and sends it back as a part of json response
-func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
-	user, err := rest.GetUserInfo(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	siteID := r.URL.Query().Get("site")
-
-	claims := auth.CustomClaims{
-		SiteID: siteID,
-		StandardClaims: jwt.StandardClaims{
-			Issuer:    "remark42",
-			ExpiresAt: time.Now().AddDate(0, 3, 0).Unix(),
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-		},
-		User: &user,
-	}
-
-	tokenStr, err := s.Authenticator.JWTService.Token(&claims)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make token")
-		return
-	}
-	link := fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", s.RemarkURL, tokenStr)
-	render.JSON(w, r, JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
-}

app/rest/api/rest_private_test.go

@@ -1,385 +0,0 @@
-package api
-
-import (
-	"compress/gzip"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestRest_Create(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	loc := c["locator"].(map[string]interface{})
-	assert.Equal(t, "radio-t", loc["site"])
-	assert.Equal(t, "https://radio-t.com/blah1", loc["url"])
-	assert.True(t, len(c["id"].(string)) > 8)
-}
-
-func TestRest_CreateOldPost(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// make old, but not too old comment
-	old := store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(old)
-	assert.Nil(t, err)
-
-	comments, err := srv.DataService.Find(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments))
-
-	// try to add new comment to the same old post
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-
-	assert.Nil(t, srv.DataService.DeleteAll("radio-t"))
-	// make too old comment
-	old = store.Comment{Text: "test test old", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(old)
-	assert.Nil(t, err)
-
-	resp, err = post(t, ts.URL+"/api/v1/comment",
-		`{"text": "test 123", "locator":{"site": "radio-t","url": "https://radio-t.com/blah1"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-}
-
-func TestRest_CreateTooBig(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	longComment := fmt.Sprintf(`{"text": "%4001s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
-
-	resp, err := post(t, ts.URL+"/api/v1/comment", longComment)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	assert.Equal(t, "comment text exceeded max allowed size 4000 (4001)", c["error"])
-	assert.Equal(t, "invalid comment", c["details"])
-
-	veryLongComment := fmt.Sprintf(`{"text": "%70000s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, "Щ")
-	resp, err = post(t, ts.URL+"/api/v1/comment", veryLongComment)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c = JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-	assert.Equal(t, "http: request body too large", c["error"])
-	assert.Equal(t, "can't bind comment", c["details"])
-}
-
-func TestRest_CreateRejected(t *testing.T) {
-
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-	body := `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`
-
-	// try to create without auth
-	resp, err := http.Post(ts.URL+"/api/v1/comment", "", strings.NewReader(body))
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-}
-func TestRest_CreateAndGet(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// create comment
-	resp, err := post(t, ts.URL+"/api/v1/comment",
-		`{"text": "**test** *123* http://radio-t.com", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	require.Nil(t, err)
-	require.Equal(t, http.StatusCreated, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	c := JSON{}
-	err = json.Unmarshal(b, &c)
-	assert.Nil(t, err)
-
-	id := c["id"].(string)
-
-	// get created comment by id
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
-	assert.Equal(t, 200, code)
-	comment := store.Comment{}
-	err = json.Unmarshal([]byte(res), &comment)
-	assert.Nil(t, err)
-	assert.Equal(t, `<p><strong>test</strong> <em>123</em> <a href="http://radio-t.com" rel="nofollow">http://radio-t.com</a></p>`+"\n", comment.Text)
-	assert.Equal(t, "**test** *123* http://radio-t.com", comment.Orig)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: "dbc7c999343f003f189f70aaf52cc04443f90790"},
-		comment.User)
-	t.Logf("%+v", comment)
-}
-
-func TestRest_Update(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id := addComment(t, c1, ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id+"?site=radio-t&url=https://radio-t.com/blah1",
-		strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err := client.Do(req)
-	assert.Nil(t, err)
-	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, b.StatusCode, string(body))
-
-	// comments returned by update
-	c2 := store.Comment{}
-	err = json.Unmarshal(body, &c2)
-	assert.Nil(t, err)
-	assert.Equal(t, id, c2.ID)
-	assert.Equal(t, "<p>updated text</p>\n", c2.Text)
-	assert.Equal(t, "updated text", c2.Orig)
-	assert.Equal(t, "my edit", c2.Edit.Summary)
-	assert.True(t, time.Since(c2.Edit.Timestamp) < 1*time.Second)
-
-	// read updated comment
-	res, code := getWithAuth(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah1", ts.URL, id))
-	assert.Equal(t, 200, code)
-	c3 := store.Comment{}
-	err = json.Unmarshal([]byte(res), &c3)
-	assert.Nil(t, err)
-	assert.Equal(t, c2, c3, "same as response from update")
-}
-
-func TestRest_UpdateNotOwner(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "xyz"}}
-	id1, err := srv.DataService.Create(c1)
-	assert.Nil(t, err)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`{"text":"updated text", "summary":"my edit"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err := client.Do(req)
-	assert.Nil(t, err)
-	body, err := ioutil.ReadAll(b.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, 403, b.StatusCode, string(body), "update from non-owner")
-	assert.Equal(t, `{"details":"can not edit comments for other users","error":"rejected"}`+"\n", string(body))
-
-	client = http.Client{}
-	req, err = http.NewRequest(http.MethodPut, ts.URL+"/api/v1/comment/"+id1+
-		"?site=radio-t&url=https://radio-t.com/blah1", strings.NewReader(`ERRR "text":"updated text", "summary":"my"}`))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	b, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 400, b.StatusCode, string(body), "update is not json")
-}
-
-func TestRest_Vote(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}}
-
-	id1 := addComment(t, c1, ts)
-	addComment(t, c2, ts)
-
-	vote := func(val int) int {
-		client := http.Client{}
-		req, err := http.NewRequest(http.MethodPut,
-			fmt.Sprintf("%s/api/v1/vote/%s?site=radio-t&url=https://radio-t.com/blah&vote=%d", ts.URL, id1, val), nil)
-		assert.Nil(t, err)
-		req.SetBasicAuth("dev", "password")
-		resp, err := client.Do(req)
-		assert.Nil(t, err)
-		return resp.StatusCode
-	}
-
-	assert.Equal(t, 200, vote(1), "first vote allowed")
-	assert.Equal(t, 400, vote(1), "second vote rejected")
-	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr := store.Comment{}
-	err := json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, cr.Score)
-	assert.Equal(t, map[string]bool{"dev": true}, cr.Votes)
-
-	assert.Equal(t, 200, vote(-1), "opposite vote allowed")
-	body, code = get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1))
-	assert.Equal(t, 200, code)
-	cr = store.Comment{}
-	err = json.Unmarshal([]byte(body), &cr)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, cr.Score)
-	assert.Equal(t, map[string]bool{}, cr.Votes)
-}
-
-func TestRest_UserAllData(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	// write 3 comments
-	user := store.User{ID: "dev", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
-	assert.Equal(t, 3, strings.Count(string(ungzBody), `"text":`), "3 comments inside")
-	t.Logf("%s", string(ungzBody))
-
-	parsed := struct {
-		Info     store.User      `json:"info"`
-		Comments []store.Comment `json:"comments"`
-	}{}
-
-	err = json.Unmarshal(ungzBody, &parsed)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev", Picture: "/api/v1/avatar/remark.image", Admin: true}, parsed.Info)
-	assert.Equal(t, 3, len(parsed.Comments))
-
-	req, err = http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	resp, err = client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 401, resp.StatusCode)
-}
-
-func TestRest_UserAllDataManyComments(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	user := store.User{ID: "dev", Name: "user name 1"}
-	c := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-
-	for i := 0; i < 478; i++ {
-		c.ID = fmt.Sprintf("id-%03d", i)
-		c.Timestamp = c.Timestamp.Add(time.Second)
-		_, err := srv.DataService.Create(c)
-		require.Nil(t, err)
-	}
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/api/v1/userdata?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
-	require.Equal(t, 200, resp.StatusCode)
-	require.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
-
-	ungzReader, err := gzip.NewReader(resp.Body)
-	assert.NoError(t, err)
-	ungzBody, err := ioutil.ReadAll(ungzReader)
-	assert.NoError(t, err)
-	assert.True(t, strings.HasPrefix(string(ungzBody),
-		`{"info": {"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","admin":true}, "comments":[{`))
-	assert.Equal(t, 478, strings.Count(string(ungzBody), `"text":`), "478 comments inside")
-}
-
-func TestRest_DeleteMe(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	m := map[string]string{}
-	err = json.Unmarshal(body, &m)
-	assert.Nil(t, err)
-	assert.Equal(t, "radio-t", m["site"])
-	assert.Equal(t, "dev", m["user_id"])
-
-	token := m["token"]
-	claims, err := srv.Authenticator.JWTService.Parse(token)
-	assert.Nil(t, err)
-	assert.Equal(t, "dev", claims.User.ID)
-	assert.Equal(t, "https://demo.remark42.com/api/v1/admin/deleteme?token="+token, m["link"])
-
-	req, err = http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/deleteme?site=radio-t", ts.URL), nil)
-	assert.Nil(t, err)
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, 401, resp.StatusCode)
-}

app/rest/api/rest_public.go

@@ -1,311 +0,0 @@
-package api
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	blackfriday "gopkg.in/russross/blackfriday.v2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-)
-
-// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score]
-// find comments for given post. Returns in tree or plain formats, sorted
-func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	sort := r.URL.Query().Get("sort")
-	if strings.HasPrefix(sort, " ") { // restore + replaced by " "
-		sort = "+" + sort[1:]
-	}
-	log.Printf("[DEBUG] get comments for %+v, sort %s, format %s", locator, sort, r.URL.Query().Get("format"))
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, sort)
-		if e != nil {
-			return nil, e
-		}
-		maskedComments := s.adminService.alterComments(comments, r)
-		var b []byte
-		switch r.URL.Query().Get("format") {
-		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
-			if s.DataService.IsReadOnly(locator) {
-				tree.Info.ReadOnly = true
-			}
-			b, e = encodeJSONWithHTML(tree)
-		default:
-			withInfo := commentsWithInfo{Comments: maskedComments}
-			if info, ee := s.DataService.Info(locator, s.ReadOnlyAge); ee == nil {
-				withInfo.Info = info
-			}
-			b, e = encodeJSONWithHTML(withInfo)
-		}
-		return b, e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// POST /preview, body is a comment, returns rendered html
-func (s *Rest) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
-	comment := store.Comment{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't bind comment")
-		return
-	}
-
-	user, err := rest.GetUserInfo(r)
-	if err != nil { // this not suppose to happen (handled by Auth), just dbl-check
-		rest.SendErrorJSON(w, r, http.StatusUnauthorized, err, "can't get user info")
-		return
-	}
-	comment.User = user
-	comment.Orig = comment.Text
-	if err = s.DataService.ValidateComment(&comment); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment")
-		return
-	}
-
-	//comment.Text = string(blackfriday.Run([]byte(comment.Text),
-	//	blackfriday.WithRenderer(bfchroma.NewRenderer(bfchroma.WithoutAutodetect()))))
-	comment.Text = string(blackfriday.Run([]byte(comment.Text), blackfriday.WithExtensions(mdExt)))
-	comment.Text = s.ImageProxy.Convert(comment.Text)
-	comment.Sanitize()
-	render.HTML(w, r, comment.Text)
-}
-
-// GET /info?site=siteID&url=post-url - get info about the post
-func (s *Rest) infoCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		info, e := s.DataService.Info(locator, s.ReadOnlyAge)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(info)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info")
-		return
-	}
-
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /last/{limit}?site=siteID - last comments for the siteID, across all posts, sorted by time
-func (s *Rest) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[DEBUG] get last comments for %s", siteID)
-
-	limit, err := strconv.Atoi(chi.URLParam(r, "limit"))
-	if err != nil {
-		limit = 0
-	}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), "last", siteID), func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, limit)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-
-		// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
-		filterDeleted := []store.Comment{}
-		for _, c := range comments {
-			if c.Deleted {
-				continue
-			}
-			filterDeleted = append(filterDeleted, c)
-		}
-
-		return encodeJSONWithHTML(filterDeleted)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /id/{id}?site=siteID&url=post-url - gets a comment by id
-func (s *Rest) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
-
-	id := chi.URLParam(r, "id")
-	siteID := r.URL.Query().Get("site")
-	url := r.URL.Query().Get("url")
-
-	log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
-
-	comment, err := s.DataService.Get(store.Locator{SiteID: siteID, URL: url}, id)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id")
-		return
-	}
-	comment = s.adminService.alterComments([]store.Comment{comment}, r)[0]
-	render.Status(r, http.StatusOK)
-	renderJSONWithHTML(w, r, comment)
-}
-
-// GET /comments?site=siteID&user=id - returns comments for given userID
-func (s *Rest) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-
-	userID := r.URL.Query().Get("user")
-	siteID := r.URL.Query().Get("site")
-
-	limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
-	if err != nil {
-		limit = 0
-	}
-
-	resp := struct {
-		Comments []store.Comment `json:"comments,omitempty"`
-		Count    int             `json:"count,omitempty"`
-	}{}
-
-	log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), userID, siteID), func() ([]byte, error) {
-		comments, e := s.DataService.User(siteID, userID, limit, 0)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-		comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
-		count, e := s.DataService.UserCount(siteID, userID)
-		if e != nil {
-			return nil, e
-		}
-		resp.Comments, resp.Count = comments, count
-		return encodeJSONWithHTML(resp)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id")
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /config?site=siteID - returns configuration
-func (s *Rest) configCtrl(w http.ResponseWriter, r *http.Request) {
-	type config struct {
-		Version        string   `json:"version"`
-		EditDuration   int      `json:"edit_duration"`
-		MaxCommentSize int      `json:"max_comment_size"`
-		Admins         []string `json:"admins"`
-		AdminEmail     string   `json:"admin_email"`
-		Auth           []string `json:"auth_providers"`
-		LowScore       int      `json:"low_score"`
-		CriticalScore  int      `json:"critical_score"`
-		ReadOnlyAge    int      `json:"readonly_age"`
-	}
-
-	cnf := config{
-		Version:        s.Version,
-		EditDuration:   int(s.DataService.EditDuration.Seconds()),
-		MaxCommentSize: s.DataService.MaxCommentSize,
-		Admins:         s.Authenticator.Admins,
-		AdminEmail:     s.Authenticator.AdminEmail,
-		LowScore:       s.ScoreThresholds.Low,
-		CriticalScore:  s.ScoreThresholds.Critical,
-		ReadOnlyAge:    s.ReadOnlyAge,
-	}
-
-	cnf.Auth = []string{}
-	for _, ap := range s.Authenticator.Providers {
-		cnf.Auth = append(cnf.Auth, ap.Name)
-	}
-
-	if cnf.Admins == nil { // prevent json serialization to nil
-		cnf.Admins = []string{}
-	}
-	render.Status(r, http.StatusOK)
-	render.JSON(w, r, cnf)
-}
-
-// GET /count?site=siteID&url=post-url - get number of comments for given post
-func (s *Rest) countCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	count, err := s.DataService.Count(locator)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count")
-		return
-	}
-	render.JSON(w, r, JSON{"count": count, "locator": locator})
-}
-
-// POST /count?site=siteID - get number of comments for posts from post body
-func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	posts := []string{}
-	if err := render.DecodeJSON(http.MaxBytesReader(w, r.Body, hardBodyLimit), &posts); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request")
-		return
-	}
-
-	// key could be long for multiple posts, make it sha1
-	key := cache.URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New()
-	if _, err := hasher.Write([]byte(key)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls")
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
-
-	data, err := s.Cache.Get(cache.Key(sha, siteID), func() ([]byte, error) {
-		counts, e := s.DataService.Counts(siteID, posts)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(counts)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID)
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}
-
-// GET /list?site=siteID&limit=50&skip=10 - list posts with comments
-func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
-
-	siteID := r.URL.Query().Get("site")
-	limit, skip := 0, 0
-
-	if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil {
-		limit = v
-	}
-	if v, err := strconv.Atoi(r.URL.Query().Get("skip")); err == nil {
-		skip = v
-	}
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID), func() ([]byte, error) {
-		posts, e := s.DataService.List(siteID, limit, skip)
-		if e != nil {
-			return nil, e
-		}
-		return encodeJSONWithHTML(posts)
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID)
-		return
-	}
-	renderJSONFromBytes(w, r, data)
-}

app/rest/api/rest_public_test.go

@@ -1,449 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestRest_Ping(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	res, code := get(t, ts.URL+"/api/v1/ping")
-	assert.Equal(t, "pong", res)
-	assert.Equal(t, 200, code)
-}
-
-func TestRest_Preview(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/preview", `{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "<p>test 123</p>\n", string(b))
-}
-
-func TestRest_PreviewWithMD(t *testing.T) {
-	srv, ts := prep(t)
-	require.NotNil(t, srv)
-	defer cleanup(ts)
-
-	text := `
-# h1
-
-BKT
-func TestRest_Preview(t *testing.T) {
-srv, ts := prep(t)
-  require.NotNil(t, srv)
-}
-BKT
-`
-	text = strings.Replace(text, "BKT", "```", -1)
-	j := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
-	j = strings.Replace(j, "\n", "\\n", -1)
-	t.Log(j)
-
-	resp, err := post(t, ts.URL+"/api/v1/preview", j)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	b, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "<h1>h1</h1>\n\n<pre><code>func TestRest_Preview(t *testing.T) {\nsrv, ts := prep(t)\n  require.NotNil(t, srv)\n}\n</code></pre>\n", string(b))
-}
-
-func TestRest_Find(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	_, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code, "nothing in")
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id1 := addComment(t, c1, ts)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: id1,
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	id2 := addComment(t, c2, ts)
-
-	assert.NotEqual(t, id1, id2)
-
-	// get sorted by +time
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=+time")
-	assert.Equal(t, 200, code)
-	comments := commentsWithInfo{}
-	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, id1, comments.Comments[0].ID)
-	assert.Equal(t, id2, comments.Comments[1].ID)
-	assert.Equal(t, "https://radio-t.com/blah1", comments.Info.URL)
-	assert.Equal(t, 2, comments.Info.Count)
-	assert.Equal(t, false, comments.Info.ReadOnly)
-	assert.True(t, comments.Info.FirstTS.Before(comments.Info.LastTS))
-
-	// get sorted by -time
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&sort=-time")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments.Comments), "should have 2 comments")
-	assert.Equal(t, id1, comments.Comments[1].ID)
-	assert.Equal(t, id2, comments.Comments[0].ID)
-
-	// get in tree mode
-	tree := rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(tree.Nodes))
-	assert.Equal(t, 1, len(tree.Nodes[0].Replies))
-	assert.Equal(t, 2, tree.Info.Count)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is fresh")
-}
-
-func TestRest_FindAge(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -5),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -15),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-
-	tree := rest.Tree{}
-
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is fresh")
-
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
-	assert.True(t, tree.Info.ReadOnly, "post is old")
-}
-
-func TestRest_FindReadOnly(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -1),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, User: store.User{ID: "u1"}}
-	_, err := srv.DataService.Create(c1)
-
-	require.Nil(t, err)
-
-	c2 := store.Comment{Text: "test test #2", ParentID: "", Timestamp: time.Now().AddDate(0, 0, -2),
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}, User: store.User{ID: "u1"}}
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-
-	// set post to read-only
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPut,
-		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah1&ro=1", ts.URL), nil)
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
-
-	tree := rest.Tree{}
-	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
-	assert.True(t, tree.Info.ReadOnly, "post is ro")
-
-	tree = rest.Tree{}
-	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &tree)
-	require.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", tree.Info.URL)
-	assert.False(t, tree.Info.ReadOnly, "post is writable")
-}
-
-func TestRest_Last(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	// add 3 comments
-	addComment(t, c1, ts)
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/last/2?site=radio-t")
-	assert.Equal(t, 200, code)
-	comments := []store.Comment{}
-	err := json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
-	assert.Equal(t, id1, comments[1].ID)
-	assert.Equal(t, id2, comments[0].ID)
-
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-
-	res, code = get(t, ts.URL+"/api/v1/last/X?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(comments), "should have 3 comments")
-
-	err = srv.DataService.Delete(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}, id1, store.SoftDelete)
-	assert.Nil(t, err)
-	res, code = get(t, ts.URL+"/api/v1/last/5?site=radio-t")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(res), &comments)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(comments), "should have 2 comments")
-}
-
-func TestRest_FindUserComments(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #3", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	// add 3 comments
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	// add one deleted
-	id := addComment(t, c2, ts)
-	err := srv.DataService.Delete(c2.Locator, id, store.SoftDelete)
-	assert.NoError(t, err)
-
-	_, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=blah")
-	assert.Equal(t, 400, code, "noting for user blah")
-
-	res, code := get(t, ts.URL+"/api/v1/comments?site=radio-t&user=dev")
-	assert.Equal(t, 200, code)
-
-	resp := struct {
-		Comments []store.Comment
-		Count    int
-	}{}
-
-	err = json.Unmarshal([]byte(res), &resp)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(resp.Comments), "should have 3 comments")
-	assert.Equal(t, 4, resp.Count, "should have 3 count")
-}
-
-func TestRest_UserInfo(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := getWithAuth(t, ts.URL+"/api/v1/user?site=radio-t")
-	assert.Equal(t, 200, code)
-	user := store.User{}
-	err := json.Unmarshal([]byte(body), &user)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "developer one", ID: "dev",
-		Picture: "/api/v1/avatar/remark.image", Admin: true, Blocked: false, IP: ""}, user)
-}
-
-func TestRest_Count(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	body, code := get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	j := JSON{}
-	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 3.0, j["count"])
-
-	body, code = get(t, ts.URL+"/api/v1/count?site=radio-t&url=https://radio-t.com/blah2")
-	assert.Equal(t, 200, code)
-	err = json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 2.0, j["count"])
-}
-
-func TestRest_Counts(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	resp, err := post(t, ts.URL+"/api/v1/counts?site=radio-t", `["https://radio-t.com/blah1","https://radio-t.com/blah2"]`)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	j := []store.PostInfo{}
-	err = json.Unmarshal(body, &j)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo([]store.PostInfo{{URL: "https://radio-t.com/blah1", Count: 3},
-		{URL: "https://radio-t.com/blah2", Count: 2}}), j)
-}
-
-func TestRest_List(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	c1 := store.Comment{Text: "test test #1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
-	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
-		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
-
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c1, ts)
-	addComment(t, c2, ts)
-	addComment(t, c2, ts)
-
-	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t")
-	assert.Equal(t, 200, code)
-	pi := []store.PostInfo{}
-	err := json.Unmarshal([]byte(body), &pi)
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
-	assert.Equal(t, 2, pi[0].Count)
-	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
-	assert.Equal(t, 3, pi[1].Count)
-}
-
-func TestRest_Config(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/api/v1/config?site=radio-t")
-	assert.Equal(t, 200, code)
-	j := JSON{}
-	err := json.Unmarshal([]byte(body), &j)
-	assert.Nil(t, err)
-	assert.Equal(t, 300., j["edit_duration"])
-	assert.EqualValues(t, []interface{}([]interface{}{"a1", "a2"}), j["admins"])
-	assert.Equal(t, "admin@remark-42.com", j["admin_email"])
-	assert.Equal(t, 4000., j["max_comment_size"])
-	assert.Equal(t, -5., j["low_score"])
-	assert.Equal(t, -10., j["critical_score"])
-	assert.Equal(t, 10., j["readonly_age"])
-	t.Logf("%+v", j)
-}
-
-func TestRest_Info(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	srv.ReadOnlyAge = 10000000 // make sure we don't hit read-only
-
-	user := store.User{ID: "user1", Name: "user name 1"}
-	c1 := store.Comment{User: user, Text: "test test #1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local)}
-	c2 := store.Comment{User: user, Text: "test test #2", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 20, 0, time.Local)}
-	c3 := store.Comment{User: user, Text: "test test #3", ParentID: "p1", Locator: store.Locator{SiteID: "radio-t",
-		URL: "https://radio-t.com/blah1"}, Timestamp: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-
-	_, err := srv.DataService.Create(c1)
-	require.Nil(t, err, "%+v", err)
-	_, err = srv.DataService.Create(c2)
-	require.Nil(t, err)
-	_, err = srv.DataService.Create(c3)
-	require.Nil(t, err)
-
-	body, code := get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-
-	info := store.PostInfo{}
-	err = json.Unmarshal([]byte(body), &info)
-	assert.Nil(t, err)
-	exp := store.PostInfo{URL: "https://radio-t.com/blah1", Count: 3,
-		FirstTS: time.Date(2018, 05, 27, 1, 14, 10, 0, time.Local), LastTS: time.Date(2018, 05, 27, 1, 14, 25, 0, time.Local)}
-	assert.Equal(t, exp, info)
-
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t&url=https://radio-t.com/blah-no")
-	assert.Equal(t, 400, code)
-	_, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no")
-	assert.Equal(t, 400, code)
-}
-
-func TestRest_Robots(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/robots.txt")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
-		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
-}

app/rest/api/rest_test.go

@@ -1,146 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/migrator"
-	"github.com/umputun/remark/app/rest/auth"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-	"github.com/umputun/remark/app/store/service"
-)
-
-var testDb = "/tmp/test-remark.db"
-var testHTML = "/tmp/test-remark.html"
-
-func TestRest_FileServer(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	body, code := get(t, ts.URL+"/web/test-remark.html")
-	assert.Equal(t, 200, code)
-	assert.Equal(t, "some html", body)
-}
-
-func TestRest_Shutdown(t *testing.T) {
-	srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300),
-		RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		srv.Shutdown()
-	}()
-
-	st := time.Now()
-	srv.Run(0)
-	assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms")
-}
-
-func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	dataStore := &service.DataStore{Interface: b, EditDuration: 5 * time.Minute, MaxCommentSize: 4000, Secret: "123456"}
-	srv = &Rest{
-		DataService: dataStore,
-		Authenticator: auth.Authenticator{
-			DevPasswd:  "password",
-			Providers:  nil,
-			Admins:     []string{"a1", "a2"},
-			AdminEmail: "admin@remark-42.com",
-			JWTService: auth.NewJWT("12345", false, time.Minute),
-		},
-		Exporter:    &migrator.Remark{DataStore: dataStore},
-		Cache:       &mockCache{},
-		WebRoot:     "/tmp",
-		RemarkURL:   "https://demo.remark42.com",
-		AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300), RoutePath: "/api/v1/avatar"},
-		ImageProxy:  &proxy.Image{},
-		ReadOnlyAge: 10,
-	}
-	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = -5, -10
-
-	err = ioutil.WriteFile(testHTML, []byte("some html"), 0700)
-	assert.Nil(t, err)
-	ts = httptest.NewServer(srv.routes())
-	return srv, ts
-}
-
-func get(t *testing.T, url string) (string, int) {
-	r, err := http.Get(url)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	require.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func getWithAuth(t *testing.T, url string) (string, int) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", url, nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	r, err := client.Do(req)
-	require.Nil(t, err)
-	defer r.Body.Close()
-	body, err := ioutil.ReadAll(r.Body)
-	assert.Nil(t, err)
-	return string(body), r.StatusCode
-}
-
-func post(t *testing.T, url string, body string) (*http.Response, error) {
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", url, strings.NewReader(body))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	return client.Do(req)
-}
-
-func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
-
-	b, err := json.Marshal(c)
-	assert.Nil(t, err, "can't marshal comment %+v", c)
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("dev", "password")
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
-	assert.Equal(t, http.StatusCreated, resp.StatusCode)
-	b, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-
-	crResp := JSON{}
-	err = json.Unmarshal(b, &crResp)
-	assert.Nil(t, err)
-	time.Sleep(time.Nanosecond * 10)
-	return crResp["id"].(string)
-}
-
-func cleanup(ts *httptest.Server) {
-	ts.Close()
-	os.Remove(testDb)
-	os.Remove(testHTML)
-}
-
-type mockCache struct{}
-
-func (mc *mockCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
-	return fn()
-}
-
-func (mc *mockCache) Flush(scopes ...string) {}

app/rest/api/rss.go

@@ -1,129 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/gorilla/feeds"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/cache"
-	"github.com/umputun/remark/app/store"
-)
-
-const maxRssItems = 20
-
-// ui uses links like <post-url>#remark42__comment-<comment-id>
-const uiNav = "#remark42__comment-"
-
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	return router
-}
-
-// GET /rss/post?site=siteID&url=post-url
-func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
-	log.Printf("[DEBUG] get rss for post %+v", locator)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), locator.SiteID, locator.URL), func() ([]byte, error) {
-		comments, e := s.DataService.Find(locator, "-time")
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-		rss, e := s.toRssFeed(locator.URL, comments)
-		if e != nil {
-			return nil, e
-		}
-		return []byte(rss), e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments")
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-// GET /rss/site?site=siteID
-func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
-	siteID := r.URL.Query().Get("site")
-	log.Printf("[DEBUG] get rss for site %s", siteID)
-
-	data, err := s.Cache.Get(cache.Key(cache.URLKey(r), siteID, "last"), func() ([]byte, error) {
-		comments, e := s.DataService.Last(siteID, maxRssItems)
-		if e != nil {
-			return nil, e
-		}
-		comments = s.adminService.alterComments(comments, r)
-
-		rss, e := s.toRssFeed(r.URL.Query().Get("site"), comments)
-		if e != nil {
-			return nil, e
-		}
-		return []byte(rss), e
-	})
-
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get last comments")
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
-		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
-	}
-}
-
-func (s *Rest) toRssFeed(url string, comments []store.Comment) (string, error) {
-
-	lastCommentTS := time.Unix(0, 0)
-	if len(comments) > 0 {
-		lastCommentTS = comments[0].Timestamp
-	}
-
-	feed := &feeds.Feed{
-		Title:       "Remark42 comments",
-		Link:        &feeds.Link{Href: url},
-		Description: "comment updates",
-		Created:     lastCommentTS,
-	}
-
-	feed.Items = []*feeds.Item{}
-	for i, c := range comments {
-		f := feeds.Item{
-			Title:       c.User.Name,
-			Link:        &feeds.Link{Href: c.Locator.URL + uiNav + c.ID},
-			Description: c.Text,
-			Created:     c.Timestamp,
-			Author:      &feeds.Author{Name: c.User.Name},
-		}
-		if c.ParentID != "" {
-			// add indication to parent comment
-			parentComment, err := s.DataService.Get(c.Locator, c.ParentID)
-			if err == nil {
-				f.Title = fmt.Sprintf("%s > %s", c.User.Name, parentComment.User.Name)
-			} else {
-				log.Printf("[WARN] failed to get info about parent comment, %s", err)
-			}
-		}
-		feed.Items = append(feed.Items, &f)
-		if i > maxRssItems {
-			break
-		}
-	}
-	return feed.ToRss()
-}

app/rest/api/rss_test.go

@@ -1,181 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestServer_RssPost(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah1", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah1")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-        <channel>
-            <title>Remark42 comments</title>
-            <link>https://radio-t.com/blah1</link>
-            <description>comment updates</description>
-            <pubDate>%s</pubDate>
-            <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah1#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-            </item>
-         </channel>
-	</rss>`, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/post?site=radio-t-bad&url=https://radio-t.com/blah1")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssSite(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah11", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/site?site=radio-t")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>radio-t</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah11#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-
-	_, code = get(t, ts.URL+"/api/v1/rss/site?site=bad-radio-t")
-	assert.Equal(t, 400, code)
-}
-
-func TestServer_RssWithReply(t *testing.T) {
-	srv, ts := prep(t)
-	assert.NotNil(t, srv)
-	defer cleanup(ts)
-
-	waitOnSecChange()
-
-	pubDate := time.Now().Format(time.RFC1123Z)
-
-	c1 := store.Comment{
-		Text:    "test 123",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	c2 := store.Comment{
-		Text:    "xyz test",
-		Locator: store.Locator{URL: "https://radio-t.com/blah10", SiteID: "radio-t"},
-	}
-	id1 := addComment(t, c1, ts)
-	c2.ParentID = id1
-	id2 := addComment(t, c2, ts)
-
-	res, code := get(t, ts.URL+"/api/v1/rss/post?site=radio-t&url=https://radio-t.com/blah10")
-	assert.Equal(t, 200, code)
-	t.Log(res)
-
-	expected := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
-		  <channel>
-		    <title>Remark42 comments</title>
-		    <link>https://radio-t.com/blah10</link>
-		    <description>comment updates</description>
-		    <pubDate>%s</pubDate>
-		    <item>
-		      <title>developer one &gt; developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;xyz test&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		    <item>
-		      <title>developer one</title>
-		      <link>https://radio-t.com/blah10#remark42__comment-%s</link>
-		      <description>&lt;p&gt;test 123&lt;/p&gt;&#xA;</description>
-		      <author>developer one</author>
-		      <pubDate>%s</pubDate>
-		    </item>
-		  </channel>
-		</rss>`, pubDate, id2, pubDate, id1, pubDate)
-
-	expected, res = cleanRssFormatting(expected, res)
-	assert.Equal(t, expected, res)
-}
-
-func waitOnSecChange() {
-	for {
-		if time.Now().Nanosecond() < 100000000 {
-			break
-		}
-		time.Sleep(10 * time.Nanosecond)
-	}
-}
-
-// clean formatting, i.e. multiple spaces, \t, \n
-func cleanRssFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/auth/auth.go

@@ -1,132 +0,0 @@
-// Package auth provides oauth2 support as well as related middlewares.
-package auth
-
-import (
-	"encoding/base64"
-	"log"
-	"net/http"
-	"strings"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Authenticator is top level auth object providing middlewares
-type Authenticator struct {
-	JWTService *JWT
-	Providers  []Provider
-	Admins     []string
-	AdminEmail string
-	DevPasswd  string
-}
-
-var devUser = store.User{
-	ID:      "dev",
-	Name:    "developer one",
-	Picture: "/api/v1/avatar/remark.image",
-	Admin:   true,
-}
-
-// Auth middleware adds auth from session and populates user info
-func (a *Authenticator) Auth(reqAuth bool) func(http.Handler) http.Handler {
-
-	f := func(h http.Handler) http.Handler {
-		fn := func(w http.ResponseWriter, r *http.Request) {
-
-			if a.basicDevUser(w, r) { // fail-back to dev user if enabled
-				user := devUser
-				r = rest.SetUserInfo(r, user)
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			claims, err := a.JWTService.Get(r)
-			if err != nil && reqAuth { // in full auth lack of session causes Unauthorized
-				log.Printf("[DEBUG] failed auth, %s", err)
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if err != nil { // in anonymous mode just pass it to the next handler
-				h.ServeHTTP(w, r)
-				return
-			}
-
-			if claims.User == nil && reqAuth {
-				log.Print("[DEBUG] failed auth, no user info presented in the claim")
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-
-			if claims.User != nil { // if uinfo in token populate it to context
-				user := *claims.User
-				user.Admin = isAdmin(user.ID, a.Admins) // dbl-check for admin to reset admin flag even if token has it
-				// refresh token if it close to expiration
-				if _, err := a.JWTService.Refresh(w, r); err != nil {
-					log.Printf("[DEBUG] can't refresh jwt, %s", err)
-				}
-				r = rest.SetUserInfo(r, user)
-			}
-			h.ServeHTTP(w, r)
-		}
-		return http.HandlerFunc(fn)
-	}
-	return f
-}
-
-// AdminOnly allows access to admins
-func (a *Authenticator) AdminOnly(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-
-		user, err := rest.GetUserInfo(r)
-		if err != nil {
-			http.Error(w, "Unauthorized", http.StatusUnauthorized)
-			return
-		}
-
-		if !user.Admin {
-			http.Error(w, "Access denied", http.StatusForbidden)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	}
-	return http.HandlerFunc(fn)
-}
-
-func (a *Authenticator) basicDevUser(w http.ResponseWriter, r *http.Request) bool {
-
-	if a.DevPasswd == "" {
-		return false
-	}
-
-	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
-	if len(s) != 2 {
-		return false
-	}
-
-	b, err := base64.StdEncoding.DecodeString(s[1])
-	if err != nil {
-		return false
-	}
-
-	pair := strings.SplitN(string(b), ":", 2)
-	if len(pair) != 2 {
-		return false
-	}
-
-	if pair[0] != "dev" || pair[1] != a.DevPasswd {
-		return false
-	}
-
-	return true
-}
-
-func isAdmin(userID string, admins []string) bool {
-	for _, admin := range admins {
-		if admin == userID {
-			return true
-		}
-	}
-	return false
-}

app/rest/auth/auth_test.go

@@ -1,171 +0,0 @@
-package auth
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAuthJWTCookie(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	expiration := int(time.Duration(365 * 24 * time.Hour).Seconds())
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-
-	client := &http.Client{Timeout: 5 * time.Second}
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtValid, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "wrong id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "xsrf mismatch")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.AddCookie(&http.Cookie{Name: "JWT", Value: testJwtExpired, HttpOnly: true, Path: "/", MaxAge: expiration, Secure: false})
-	req.Header.Add("X-XSRF-TOKEN", "random id")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "token expired")
-}
-
-func TestAuthJWTHeader(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456", JWTService: NewJWT("xyz 12345", false, time.Hour)}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtValid)
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.Nil(t, err)
-	req.Header.Add("X-JWT", testJwtExpired)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "invalid auth token")
-}
-func TestAuthRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "xyz")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 401, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAuthNotRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(false)).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "no auth user")
-
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "ZZZZ123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "wrong auth creds")
-}
-
-func TestAdminRequired(t *testing.T) {
-	a := Authenticator{DevPasswd: "123456"}
-	router := chi.NewRouter()
-	router.With(a.Auth(true), a.AdminOnly).Get("/auth", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(201)
-	})
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err := client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 201, resp.StatusCode, "valid auth user, admin")
-
-	devUser.Admin = false
-	req, err = http.NewRequest("GET", server.URL+"/auth", nil)
-	require.NoError(t, err)
-	req = withBasicAuth(req, "dev", "123456")
-	resp, err = client.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, 403, resp.StatusCode, "valid auth user, not admin")
-
-}
-func withBasicAuth(r *http.Request, username, password string) *http.Request {
-	auth := username + ":" + password
-	r.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
-	return r
-}

app/rest/auth/jwt.go

@@ -1,166 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"time"
-
-	jwt "github.com/dgrijalva/jwt-go"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// JWT wraps jwt operations
-// supports both header and cookie jwt
-type JWT struct {
-	secret        string
-	secureCookies bool
-	exp           time.Duration
-}
-
-// CustomClaims stores user info for auth and state & from from login
-type CustomClaims struct {
-	jwt.StandardClaims
-	User *store.User `json:"user,omitempty"`
-
-	// state and from used for oauth handshake
-	State       string `json:"state,omitempty"`
-	From        string `json:"from,omitempty"`
-	SiteID      string `json:"site_id,omitempty"`
-	SessionOnly bool   `json:"sess_only,omitempty"`
-}
-
-const jwtCookieName = "JWT"
-const jwtHeaderKey = "X-JWT"
-const xsrfCookieName = "XSRF-TOKEN"
-const xsrfHeaderKey = "X-XSRF-TOKEN"
-
-// NewJWT makes JWT service
-func NewJWT(secret string, secureCookies bool, exp time.Duration) *JWT {
-	res := JWT{
-		secret:        secret,
-		secureCookies: secureCookies,
-		exp:           exp,
-	}
-	return &res
-}
-
-// Token makes jwt with claims
-func (j *JWT) Token(claims *CustomClaims) (string, error) {
-	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
-	tokenString, err := token.SignedString([]byte(j.secret))
-	if err != nil {
-		return "", errors.Wrap(err, "can't sign jwt token")
-	}
-	return tokenString, nil
-}
-
-// Parse token string and verify
-func (j *JWT) Parse(tokenString string) (*CustomClaims, error) {
-	token, err := jwt.ParseWithClaims(tokenString, &CustomClaims{}, func(token *jwt.Token) (interface{}, error) {
-		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
-			return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"])
-		}
-		return []byte(j.secret), nil
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "can't parse jwt")
-	}
-
-	claims, ok := token.Claims.(*CustomClaims)
-	if !ok || !token.Valid {
-		return nil, errors.New("invalid jwt")
-	}
-	return claims, nil
-}
-
-// Set creates jwt cookie with xsrf cookie and put it to ResponseWriter
-// accepts claims and sets expiration if none defined. permanent flag means long-living cookie, false makes it session only.
-func (j *JWT) Set(w http.ResponseWriter, claims *CustomClaims, sessionOnly bool) error {
-	if claims.ExpiresAt == 0 {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-	}
-
-	tokenString, err := j.Token(claims)
-	if err != nil {
-		return errors.Wrap(err, "failed to make jwt token")
-	}
-
-	cookieExpiration := 0 // session cookie
-	if !sessionOnly {
-		cookieExpiration = 365 * 24 * 3600 // 1 year
-	}
-
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: tokenString, HttpOnly: true, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: claims.Id, HttpOnly: false, Path: "/",
-		MaxAge: cookieExpiration, Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-
-	return nil
-}
-
-// Get jwt from header or cookie
-// if cookie used, verify xsrf token to match
-func (j *JWT) Get(r *http.Request) (*CustomClaims, error) {
-
-	fromCookie := false
-	tokenString := ""
-
-	// try to get from X-JWT header
-	if tokenHeader := r.Header.Get(jwtHeaderKey); tokenHeader != "" {
-		tokenString = tokenHeader
-	}
-
-	// try to get from JWT cookie
-	if tokenString == "" {
-		fromCookie = true
-		jc, err := r.Cookie(jwtCookieName)
-		if err != nil {
-			return nil, errors.Wrap(err, "jwt cookie was not presented")
-		}
-		tokenString = jc.Value
-	}
-
-	claims, err := j.Parse(tokenString)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get jwt")
-	}
-
-	if fromCookie && claims.User != nil {
-		xsrf := r.Header.Get(xsrfHeaderKey)
-		if claims.Id != xsrf {
-			return nil, errors.New("xsrf mismatch")
-		}
-	}
-
-	return claims, nil
-}
-
-// Refresh gets jwt from request, checks if it will be expiring soon (1/2 of expiration) and create the new onw
-func (j *JWT) Refresh(w http.ResponseWriter, r *http.Request) (*CustomClaims, error) {
-	claims, err := j.Get(r)
-	if err != nil {
-		return nil, err
-	}
-	untilExp := claims.ExpiresAt - time.Now().Unix()
-	if untilExp <= int64(j.exp.Seconds()/2) {
-		claims.ExpiresAt = time.Now().Add(j.exp).Unix()
-		e := j.Set(w, claims, claims.SessionOnly)
-		return claims, e
-	}
-	return claims, nil
-}
-
-// Reset token's cookies
-func (j *JWT) Reset(w http.ResponseWriter) {
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &jwtCookie)
-
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: "", HttpOnly: false, Path: "/",
-		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.secureCookies}
-	http.SetCookie(w, &xsrfCookie)
-}

app/rest/auth/jwt_test.go

@@ -1,287 +0,0 @@
-package auth
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-var testJwtValid = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCI" +
-	"sImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZS" +
-	"I6IiIsImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ._loFgh3g45gr9TtGqvM3N584I_6EHEOJnYb6Py84stQ"
-
-var testJwtValidSess = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI3ODkxOTE4MjIsImp0aSI6InJhbmRvbSBpZCIsImlzcyI6In" +
-	"JlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiIsIm" +
-	"FkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20iLCJzZXNzX29ubHkiOnRydWV9.p6w0sM_NYaRuyhyA9jqfWlB5cx1vZPGhXGC5geSX7nA"
-
-var testJwtExpired = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MjY4ODc4MjIsImp0aSI6InJhbmRvbSBpZCIs" +
-	"ImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTI2ODg0MjIyLCJ1c2VyIjp7Im5hbWUiOiJuYW1lMSIsImlkIjoiaWQxIiwicGljdHVyZSI6IiI" +
-	"sImFkbWluIjpmYWxzZX0sInN0YXRlIjoiMTIzNDU2IiwiZnJvbSI6ImZyb20ifQ.4_dCrY9ihyfZIedz-kZwBTxmxU1a52V7IqeJrOqTzE4"
-
-func TestJWT_Token(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	res, err := j.Token(claims)
-	assert.Nil(t, err)
-	assert.Equal(t, testJwtValid, res)
-}
-
-func TestJWT_Parse(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-	claims, err := j.Parse(testJwtValid)
-	assert.NoError(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1"}, claims.User)
-
-	_, err = j.Parse(testJwtExpired)
-	assert.NotNil(t, err, "expired token")
-
-	_, err = j.Parse("bad")
-	assert.NotNil(t, err, "bad token")
-}
-
-func TestJWT_Set(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	claims.SessionOnly = false
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValid, cookies[0].Value)
-	assert.Equal(t, 31536000, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-
-	claims.SessionOnly = true
-	rr = httptest.NewRecorder()
-	err = j.Set(rr, claims, claims.SessionOnly)
-	assert.Nil(t, err)
-	cookies = rr.Result().Cookies()
-	t.Log(cookies)
-	require.Equal(t, 2, len(cookies))
-	assert.Equal(t, "JWT", cookies[0].Name)
-	assert.Equal(t, testJwtValidSess, cookies[0].Value)
-	assert.Equal(t, 0, cookies[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", cookies[1].Name)
-	assert.Equal(t, "random id", cookies[1].Value)
-}
-
-func TestJWT_GetFromHeader(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	req := httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtValid)
-	claims, err := j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, testJwtExpired)
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-
-	req = httptest.NewRequest("GET", "/", nil)
-	req.Header.Add(jwtHeaderKey, "bad bad token")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token contains an invalid number of segments"), err.Error())
-
-}
-
-func TestJWT_SetAndGetWithCookies(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State:       "123456",
-		From:        "from",
-		SessionOnly: true,
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	claims, err = j.Get(req)
-	assert.Nil(t, err)
-	assert.Equal(t, &store.User{Name: "name1", ID: "id1", Picture: "", Admin: false, Blocked: false, IP: ""}, claims.User)
-	assert.Equal(t, "remark42", claims.Issuer)
-	assert.Equal(t, true, claims.SessionOnly)
-	t.Log(resp.Cookies())
-}
-
-func TestJWT_SetAndGetWithXsrfMismatch(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2058, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/valid" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/valid")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/valid", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id wrong")
-	claims, err = j.Get(req)
-	assert.EqualError(t, err, "xsrf mismatch")
-}
-
-func TestJWT_SetAndGetWithCookiesExpired(t *testing.T) {
-	j := NewJWT("xyz 12345", false, time.Hour)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:        "random id",
-			Issuer:    "remark42",
-			ExpiresAt: time.Date(2018, 5, 21, 1, 35, 22, 0, time.Local).Unix(),
-			NotBefore: time.Date(2018, 5, 21, 1, 30, 22, 0, time.Local).Unix(),
-		},
-	}
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/expired" {
-			assert.Nil(t, j.Set(w, claims, true))
-			w.WriteHeader(200)
-		}
-	}))
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/expired")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	req := httptest.NewRequest("GET", "/expired", nil)
-	req.AddCookie(resp.Cookies()[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-	_, err = j.Get(req)
-	assert.NotNil(t, err)
-	assert.True(t, strings.Contains(err.Error(), "can't parse jwt: token is expired by"), err.Error())
-}
-
-func TestJWT_Refresh(t *testing.T) {
-	j := NewJWT("xyz 12345", false, 2*time.Second)
-
-	claims := &CustomClaims{
-		State: "123456",
-		From:  "from",
-		User: &store.User{
-			ID:   "id1",
-			Name: "name1",
-		},
-		StandardClaims: jwt.StandardClaims{
-			Id:     "random id",
-			Issuer: "remark42",
-		},
-	}
-	// set token
-	rr := httptest.NewRecorder()
-	err := j.Set(rr, claims, true)
-	assert.Nil(t, err)
-	cookies := rr.Result().Cookies()
-	require.Equal(t, 2, len(cookies))
-
-	req, err := http.NewRequest("GET", "http://example.com/blah", nil)
-	require.Nil(t, err)
-	req.AddCookie(cookies[0])
-	req.Header.Add(xsrfHeaderKey, "random id")
-
-	claims2, err := j.Refresh(rr, req)
-	require.Nil(t, err)
-	assert.Equal(t, claims.ExpiresAt, claims2.ExpiresAt, "no refresh yet")
-
-	time.Sleep(1 * time.Second)
-	claims2, err = j.Refresh(rr, req)
-	assert.Nil(t, err)
-	assert.True(t, claims.ExpiresAt < claims2.ExpiresAt, "refreshed")
-	t.Log(claims.ExpiresAt, claims2.ExpiresAt)
-}

app/rest/auth/provider.go

@@ -1,224 +0,0 @@
-package auth
-
-import (
-	"context"
-	"crypto/rand"
-	"crypto/sha1"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/dgrijalva/jwt-go"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/render"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/rest/proxy"
-	"github.com/umputun/remark/app/store"
-)
-
-// Provider represents oauth2 provider
-type Provider struct {
-	Params
-	Name        string
-	RedirectURL string
-	InfoURL     string
-	Endpoint    oauth2.Endpoint
-	Scopes      []string
-	MapUser     func(userData, []byte) store.User // map info from InfoURL to User
-	conf        oauth2.Config
-}
-
-// Params to make initialized and ready to use provider
-type Params struct {
-	RemarkURL    string
-	AvatarProxy  *proxy.Avatar
-	JwtService   *JWT
-	IsVerifiedFn func(siteID string, userID string) bool
-	SecretKey    string
-	Admins       []string
-	Cid          string
-	Csecret      string
-}
-
-type userData map[string]interface{}
-
-func (u userData) value(key string) string {
-	// json.Unmarshal converts json "null" value to go's "nil", in this case return empty string
-	if val, ok := u[key]; ok && val != nil {
-		return fmt.Sprintf("%v", val)
-	}
-	return ""
-}
-
-// newProvider makes auth for given provider
-func initProvider(p Params, provider Provider) Provider {
-	log.Printf("[INFO] init auth provider %s", provider.Name)
-	provider.Params = p
-	provider.conf = oauth2.Config{
-		ClientID:     provider.Cid,
-		ClientSecret: provider.Csecret,
-		RedirectURL:  provider.RedirectURL,
-		Scopes:       provider.Scopes,
-		Endpoint:     provider.Endpoint,
-	}
-
-	log.Printf("[DEBUG] created %s auth, id=%s, redir=%s, endpoint=%s",
-		provider.Name, provider.Cid, provider.Endpoint, provider.RedirectURL)
-	return provider
-}
-
-// Routes returns auth routes for given provider
-func (p Provider) Routes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/login", p.loginHandler)
-	router.Get("/callback", p.authHandler)
-	router.Get("/logout", p.LogoutHandler)
-	return router
-}
-
-// loginHandler - GET /login?from=redirect-back-url&site=siteID&session=1
-func (p Provider) loginHandler(w http.ResponseWriter, r *http.Request) {
-
-	log.Printf("[DEBUG] login with %s", p.Name)
-	// make state (random) and store in session
-	state := p.randToken()
-
-	claims := CustomClaims{
-		State:       state,
-		From:        r.URL.Query().Get("from"),
-		SiteID:      r.URL.Query().Get("site"),
-		SessionOnly: r.URL.Query().Get("session") != "" && r.URL.Query().Get("session") != "0",
-		StandardClaims: jwt.StandardClaims{
-			Id:        p.randToken(),
-			Issuer:    "remark42",
-			ExpiresAt: time.Now().Add(30 * time.Minute).Unix(),
-			NotBefore: time.Now().Add(-1 * time.Minute).Unix(),
-		},
-	}
-
-	if err := p.JwtService.Set(w, &claims, false); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to set jwt")
-		return
-	}
-
-	// return login url
-	loginURL := p.conf.AuthCodeURL(state)
-	log.Printf("[DEBUG] login url %s", loginURL)
-
-	http.Redirect(w, r, loginURL, http.StatusFound)
-}
-
-// authHandler fills user info and redirects to "from" url. This is callback url redirected locally by browser
-// GET /callback
-func (p Provider) authHandler(w http.ResponseWriter, r *http.Request) {
-
-	oauthClaims, err := p.JwtService.Get(r)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to get jwt")
-		return
-	}
-
-	retrievedState := oauthClaims.State
-	if retrievedState == "" || retrievedState != r.URL.Query().Get("state") {
-		http.Error(w, fmt.Sprintf("unexpected state %v", retrievedState), http.StatusUnauthorized)
-		return
-	}
-
-	log.Printf("[DEBUG] auth with state %s", retrievedState)
-	tok, err := p.conf.Exchange(context.Background(), r.URL.Query().Get("code"))
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "exchange failed")
-		return
-	}
-
-	client := p.conf.Client(context.Background(), tok)
-	uinfo, err := client.Get(p.InfoURL)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, fmt.Sprintf("failed to get client info via %s", p.InfoURL))
-		return
-	}
-
-	defer func() {
-		if e := uinfo.Body.Close(); e != nil {
-			log.Printf("[WARN] failed to close response body, %s", e)
-		}
-	}()
-
-	data, err := ioutil.ReadAll(uinfo.Body)
-	if err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to read user info")
-		return
-	}
-
-	jData := map[string]interface{}{}
-	if e := json.Unmarshal(data, &jData); e != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to unmarshal user info")
-		return
-	}
-	log.Printf("[DEBUG] got raw user info %+v", jData)
-
-	u := p.MapUser(jData, data)
-	u = p.alterUser(u, oauthClaims)
-
-	authClaims := &CustomClaims{
-		User: &u,
-		StandardClaims: jwt.StandardClaims{
-			Issuer: "remark42",
-			Id:     p.randToken(),
-		},
-		SessionOnly: oauthClaims.SessionOnly,
-	}
-
-	if err = p.JwtService.Set(w, authClaims, oauthClaims.SessionOnly); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "failed to save user info")
-		return
-	}
-
-	log.Printf("[DEBUG] user info %+v", u)
-
-	// redirect to back url if presented in login query params
-	if oauthClaims.From != "" {
-		http.Redirect(w, r, oauthClaims.From, http.StatusTemporaryRedirect)
-		return
-	}
-	render.JSON(w, r, &u)
-}
-
-// alterUser sets fields not handled by provider's MapUser, things like avatar, admin, verified
-func (p Provider) alterUser(u store.User, oauthClaims *CustomClaims) store.User {
-	if p.AvatarProxy != nil {
-		if avatarURL, e := p.AvatarProxy.Put(u); e == nil {
-			u.Picture = avatarURL
-		} else {
-			log.Printf("[WARN] failed to proxy avatar, %s", e)
-		}
-	}
-	u.Admin = isAdmin(u.ID, p.Admins)
-	if p.IsVerifiedFn != nil {
-		u.Verified = p.IsVerifiedFn(oauthClaims.SiteID, u.ID)
-	}
-	return u
-}
-
-// LogoutHandler - GET /logout
-func (p Provider) LogoutHandler(w http.ResponseWriter, r *http.Request) {
-	p.JwtService.Reset(w)
-	log.Printf("[DEBUG] logout")
-}
-
-func (p Provider) randToken() string {
-	b := make([]byte, 32)
-	if _, err := rand.Read(b); err != nil {
-		log.Fatalf("[ERROR] can't get randoms, %s", err)
-	}
-	s := sha1.New()
-	if _, err := s.Write(b); err != nil {
-		log.Printf("[WARN] can't write randoms, %s", err)
-	}
-	return fmt.Sprintf("%x", s.Sum(nil))
-}

app/rest/auth/provider_test.go

@@ -1,218 +0,0 @@
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/http/cookiejar"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/oauth2"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestLogin(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, permanent
-	resp, err := client.Get("http://localhost:8981/login?site=remark")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	t.Logf("resp %s", string(body))
-	t.Logf("headers: %+v", resp.Header)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 31536000, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	u := store.User{}
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser1", Picture: "http://exmple.com/pic1.png",
-		Admin: false, Blocked: false, IP: ""}, u)
-
-	// check admin user
-	resp, err = client.Get("http://localhost:8981/login?site=remark")
-	assert.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err = ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	err = json.Unmarshal(body, &u)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "blah", ID: "mock_myuser2", Picture: "http://exmple.com/pic1.png",
-		Admin: true, Blocked: false, IP: "", Verified: true}, u)
-}
-
-func TestLoginSessionOnly(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8981, 8982)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	// check non-admin, session
-	resp, err := client.Get("http://localhost:8981/login?site=remark&session=1")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name)
-	assert.NotEqual(t, "", resp.Cookies()[0].Value, "jwt set")
-	assert.Equal(t, 0, resp.Cookies()[0].MaxAge)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name)
-	assert.NotEqual(t, "", resp.Cookies()[1].Value, "xsrf cookie set")
-
-	req, err := http.NewRequest("GET", "http://example.com", nil)
-	require.Nil(t, err)
-	req.AddCookie(resp.Cookies()[0])
-	req.AddCookie(resp.Cookies()[1])
-	req.Header.Add("X-XSRF-TOKEN", resp.Cookies()[1].Value)
-
-	jwtService := NewJWT("12345", false, time.Hour)
-	res, err := jwtService.Get(req)
-	require.Nil(t, err)
-	assert.Equal(t, true, res.SessionOnly)
-	t.Logf("%+v", res)
-}
-
-func TestLogout(t *testing.T) {
-
-	ts, ots := mockProvider(t, 8691, 8692)
-	defer func() {
-		ts.Close()
-		ots.Close()
-	}()
-
-	jar, err := cookiejar.New(nil)
-	require.Nil(t, err)
-	client := &http.Client{Jar: jar, Timeout: 5 * time.Second}
-
-	resp, err := client.Get("http://localhost:8691/login")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	assert.Equal(t, 2, len(resp.Cookies()))
-	resp, err = client.Get("http://localhost:8691/logout")
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-
-	assert.Equal(t, 2, len(resp.Cookies()))
-	assert.Equal(t, "JWT", resp.Cookies()[0].Name, "jwt cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[0].Value)
-	assert.Equal(t, "XSRF-TOKEN", resp.Cookies()[1].Name, "xsrf cookie cleared")
-	assert.Equal(t, "", resp.Cookies()[1].Value)
-}
-
-func TestInitProvider(t *testing.T) {
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret"}
-	provider := Provider{Name: "test", RedirectURL: "redir"}
-	res := initProvider(params, provider)
-	assert.Equal(t, "cid", res.conf.ClientID)
-	assert.Equal(t, "csecret", res.conf.ClientSecret)
-	assert.Equal(t, "redir", res.RedirectURL)
-	assert.Equal(t, "123456", res.SecretKey)
-	assert.Equal(t, "test", res.Name)
-}
-
-func mockProvider(t *testing.T, loginPort, authPort int) (*http.Server, *http.Server) {
-
-	provider := Provider{
-		Name: "mock",
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  fmt.Sprintf("http://localhost:%d/login/oauth/authorize", authPort),
-			TokenURL: fmt.Sprintf("http://localhost:%d/login/oauth/access_token", authPort),
-		},
-		RedirectURL: fmt.Sprintf("http://localhost:%d/callback", loginPort),
-		Scopes:      []string{"user:email"},
-		InfoURL:     fmt.Sprintf("http://localhost:%d/user", authPort),
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:      "mock_" + data.value("id"),
-				Name:    data.value("name"),
-				Picture: data.value("picture"),
-			}
-			return userInfo
-		},
-	}
-
-	params := Params{RemarkURL: "url", SecretKey: "123456", Cid: "cid", Csecret: "csecret",
-		JwtService: NewJWT("12345", false, time.Hour), Admins: []string{"mock_myuser2"},
-		// AvatarProxy:  &proxy.Avatar{Store: &mockAvatarStore, RoutePath: "/v1/avatar"},
-		IsVerifiedFn: func(siteID, userID string) bool { return userID == "mock_myuser2" }}
-	provider = initProvider(params, provider)
-
-	ts := &http.Server{Addr: fmt.Sprintf(":%d", loginPort), Handler: provider.Routes()}
-
-	count := 0
-	useIds := []string{"myuser1", "myuser2"}
-
-	oauth := &http.Server{
-		Addr: fmt.Sprintf(":%d", authPort),
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			log.Printf("[MOCK OAUTH] request %s %s %+v", r.Method, r.URL, r.Header)
-			switch {
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/authorize"):
-				state := r.URL.Query().Get("state")
-				w.Header().Add("Location", fmt.Sprintf("http://localhost:%d/callback?code=g0ZGZmNjVmOWI&state=%s",
-					loginPort, state))
-				w.WriteHeader(302)
-			case strings.HasPrefix(r.URL.Path, "/login/oauth/access_token"):
-				res := `{
-					"access_token":"MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3",
-					"token_type":"bearer",
-					"expires_in":3600,
-					"refresh_token":"IwOGYzYTlmM2YxOTQ5MGE3YmNmMDFkNTVk",
-					"scope":"create",
-					"state":"12345678"
-					}`
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			case strings.HasPrefix(r.URL.Path, "/user"):
-				res := fmt.Sprintf(`{
-					"id": "%s",
-					"name":"blah",
-					"picture":"http://exmple.com/pic1.png"
-					}`, useIds[count])
-				count++
-				w.Header().Set("Content-Type", "application/json; charset=utf-8")
-				w.WriteHeader(200)
-				_, err := w.Write([]byte(res))
-				assert.NoError(t, err)
-			default:
-				t.Fatalf("unexpected oauth request %s %s", r.Method, r.URL)
-			}
-		}),
-	}
-
-	go func() { _ = oauth.ListenAndServe() }()
-	go func() { _ = ts.ListenAndServe() }()
-
-	time.Sleep(time.Millisecond * 100) // let them start
-	return ts, oauth
-}

app/rest/auth/providers.go

@@ -1,126 +0,0 @@
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"golang.org/x/oauth2/facebook"
-	"golang.org/x/oauth2/github"
-	"golang.org/x/oauth2/google"
-	"golang.org/x/oauth2/yandex"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// NewGoogle makes google oauth2 provider
-func NewGoogle(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "google",
-		Endpoint:    google.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/google/callback",
-		Scopes:      []string{"https://www.googleapis.com/auth/userinfo.profile"},
-		InfoURL:     "https://www.googleapis.com/oauth2/v3/userinfo",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				// encode email with provider name to avoid collision if same id returned by other provider
-				ID:      "google_" + store.EncodeID(data.value("sub")),
-				Name:    data.value("name"),
-				Picture: data.value("picture"),
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = "noname_" + userInfo.ID[8:12]
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewGithub makes github oauth2 provider
-func NewGithub(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "github",
-		Endpoint:    github.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/github/callback",
-		Scopes:      []string{},
-		InfoURL:     "https://api.github.com/user",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:      "github_" + store.EncodeID(data.value("login")),
-				Name:    data.value("name"),
-				Picture: data.value("avatar_url"),
-			}
-			// github may have no user name, use login in this case
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("login")
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewFacebook makes facebook oauth2 provider
-func NewFacebook(p Params) Provider {
-
-	// response format for fb /me call
-	type uinfo struct {
-		ID      string `json:"id"`
-		Name    string `json:"name"`
-		Picture struct {
-			Data struct {
-				URL string `json:"url"`
-			} `json:"data"`
-		} `json:"picture"`
-	}
-
-	return initProvider(p, Provider{
-		Name:        "facebook",
-		Endpoint:    facebook.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/facebook/callback",
-		Scopes:      []string{"public_profile"},
-		InfoURL:     "https://graph.facebook.com/me?fields=id,name,picture",
-		MapUser: func(data userData, bdata []byte) store.User {
-			userInfo := store.User{
-				ID:   "facebook_" + store.EncodeID(data.value("id")),
-				Name: data.value("name"),
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = userInfo.ID[0:16]
-			}
-
-			uinfoJSON := uinfo{}
-			if err := json.Unmarshal(bdata, &uinfoJSON); err == nil {
-				userInfo.Picture = uinfoJSON.Picture.Data.URL
-			}
-			return userInfo
-		},
-	})
-}
-
-// NewYandex makes yandex oauth2 provider
-func NewYandex(p Params) Provider {
-	return initProvider(p, Provider{
-		Name:        "yandex",
-		Endpoint:    yandex.Endpoint,
-		RedirectURL: p.RemarkURL + "/auth/yandex/callback",
-		Scopes:      []string{},
-		// See https://tech.yandex.com/passport/doc/dg/reference/response-docpage/
-		InfoURL: "https://login.yandex.ru/info?format=json",
-		MapUser: func(data userData, _ []byte) store.User {
-			userInfo := store.User{
-				ID:   "yandex_" + store.EncodeID(data.value("id")),
-				Name: data.value("display_name"), // using Display Name by default
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("real_name") // using Real Name (== full name) if Display Name is empty
-			}
-			if userInfo.Name == "" {
-				userInfo.Name = data.value("login") // otherwise using login
-			}
-
-			if data.value("default_avatar_id") != "" {
-				userInfo.Picture = fmt.Sprintf("https://avatars.yandex.net/get-yapic/%s/islands-200", data.value("default_avatar_id"))
-			}
-			return userInfo
-		},
-	})
-}

app/rest/auth/providers_test.go

@@ -1,83 +0,0 @@
-package auth
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestProviders_NewGoogle(t *testing.T) {
-	r := NewGoogle(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "google", r.Name)
-
-	udata := userData{"sub": "1234567890", "name": "test user", "picture": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// no name in data
-	udata = userData{"sub": "1234567890", "picture": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "noname_1b30", ID: "google_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewGithub(t *testing.T) {
-	r := NewGithub(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "github", r.Name)
-
-	udata := userData{"login": "lll", "name": "test user", "avatar_url": "http://demo.remark42.com/blah.png"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "test user", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// nil name in data (json response contains `"name": null`); using login, it's always required
-	udata = userData{"login": "lll", "name": nil, "avatar_url": "http://demo.remark42.com/blah.png"}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "lll", ID: "github_e80b2d2608711cbb3312db7c4727a46fbad9601a",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewFacebook(t *testing.T) {
-	r := NewFacebook(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "facebook", r.Name)
-
-	udata := userData{"id": "myid", "name": "test user"}
-	user := r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "test user", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	udata = userData{"id": "myid", "name": ""}
-	user = r.MapUser(udata, []byte(`{"picture": {"data": {"url": "http://demo.remark42.com/blah.png"} }}`))
-	assert.Equal(t, store.User{Name: "facebook_6e34471", ID: "facebook_6e34471f84557e1713012d64a7477c71bfdac631",
-		Picture: "http://demo.remark42.com/blah.png", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}
-
-func TestProviders_NewYandex(t *testing.T) {
-	r := NewYandex(Params{RemarkURL: "http://demo.remark42.com", Cid: "cid", Csecret: "cs"})
-	assert.Equal(t, "yandex", r.Name)
-
-	udata := userData{"id": "1234567890", "display_name": "Vasya P", "default_avatar_id": "131652443"}
-	user := r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya P", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "https://avatars.yandex.net/get-yapic/131652443/islands-200", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "display_name": null, "default_avatar_id": null
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": nil, "real_name": "Vasya Pupkin", "default_avatar_id": nil}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "Vasya Pupkin", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// empty "display_name", empty "default_avatar_id", empty "real_name"
-	udata = userData{"id": "1234567890", "login": "vasya", "display_name": "", "real_name": "", "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-
-	// "real_name": null
-	udata = userData{"id": "1234567890", "login": "vasya", "real_name": nil, "default_avatar_id": ""}
-	user = r.MapUser(udata, nil)
-	assert.Equal(t, store.User{Name: "vasya", ID: "yandex_01b307acba4f54f55aafc33bb06bbbf6ca803e9a",
-		Picture: "", Admin: false, Blocked: false, IP: ""}, user, "got %+v", user)
-}

app/rest/cache/cache.go

@@ -1,45 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
-)
-
-// LoadingCache defines interface for caching
-type LoadingCache interface {
-	Get(key string, fn func() ([]byte, error)) (data []byte, err error)
-	Flush(scopes ...string)
-}
-
-// Key makes full key from primary key and scopes
-func Key(key string, scopes ...string) string {
-	return strings.Join(scopes, "$$") + "@@" + key
-}
-
-// ParseKey gets compound key created by Key func and split it to the actual key and scopes
-func ParseKey(fullKey string) (key string, scopes []string, err error) {
-	elems := strings.Split(fullKey, "@@")
-	if len(elems) != 2 {
-		return "", nil, errors.Errorf("can't parse cache key %s", key)
-	}
-	scopes = strings.Split(elems[0], "$$")
-	if len(scopes) == 1 && scopes[0] == "" {
-		scopes = []string{}
-	}
-	key = elems[1]
-	return key, scopes, nil
-}
-
-// URLKey gets url from request to use it as cache key
-// admins will have different keys in order to prevent leak of admin-only data to regular users
-func URLKey(r *http.Request) string {
-	adminPrefix := "admin!!"
-	key := strings.TrimPrefix(r.URL.String(), adminPrefix)          // prevents attach with fake url to get admin view
-	if user, err := rest.GetUserInfo(r); err == nil && user.Admin { // make separate cache key for admins
-		key = adminPrefix + key
-	}
-	return key
-}

app/rest/cache/cache_test.go

@@ -1,54 +0,0 @@
-package cache
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestCache_Keys(t *testing.T) {
-	tbl := []struct {
-		key    string
-		scopes []string
-		full   string
-	}{
-		{"key1", []string{"s1"}, "s1@@key1"},
-		{"key2", []string{"s11", "s2"}, "s11$$s2@@key2"},
-		{"key3", []string{}, "@@key3"},
-	}
-
-	for n, tt := range tbl {
-		full := Key(tt.key, tt.scopes...)
-		assert.Equal(t, tt.full, full, "making key, #%d", n)
-
-		k, s, e := ParseKey(full)
-		assert.Nil(t, e)
-		assert.Equal(t, tt.scopes, s)
-		assert.Equal(t, tt.key, k)
-	}
-
-	_, _, err := ParseKey("abc")
-	assert.Error(t, err)
-	_, _, err = ParseKey("")
-	assert.Error(t, err)
-}
-
-func TestCache_URLKey(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah/123", nil)
-	assert.Nil(t, err)
-	key := URLKey(r)
-	assert.Equal(t, "http://blah/123", key)
-
-	r, err = http.NewRequest("GET", "http://blah/123?key=v&k2=v2", nil)
-	assert.Nil(t, err)
-	key = URLKey(r)
-	assert.Equal(t, "http://blah/123?key=v&k2=v2", key)
-
-	user := store.User{Admin: true}
-	r = rest.SetUserInfo(r, user)
-	key = URLKey(r)
-	assert.Equal(t, "admin!!http://blah/123?key=v&k2=v2", key)
-}

app/rest/cache/memory.go

@@ -1,114 +0,0 @@
-package cache
-
-import (
-	"log"
-	"sync/atomic"
-
-	"github.com/hashicorp/golang-lru"
-	"github.com/pkg/errors"
-)
-
-// memoryCache implements LoadingCache interface on top of cache.Cache (go-cache)
-type memoryCache struct {
-	bytesCache   *lru.Cache
-	postFlushFn  func()
-	maxKeys      int
-	maxValueSize int
-	maxCacheSize int64
-	currentSize  int64
-}
-
-// NewMemoryCache makes memoryCache implementation
-func NewMemoryCache(options ...Option) (LoadingCache, error) {
-	res := memoryCache{
-		postFlushFn:  func() {},
-		maxKeys:      1000,
-		maxValueSize: 0,
-	}
-	for _, opt := range options {
-		if err := opt(&res); err != nil {
-			return nil, errors.Wrap(err, "failed to set cache option")
-		}
-	}
-
-	onEvicted := func(key interface{}, value interface{}) {
-		size := len(value.([]byte))
-		atomic.AddInt64(&res.currentSize, -1*int64(size))
-	}
-
-	var err error
-	// OnEvicted called automatically for expired and manually deleted
-	if res.bytesCache, err = lru.NewWithEvict(res.maxKeys, onEvicted); err != nil {
-		return nil, errors.Wrap(err, "failed to make cache")
-	}
-
-	log.Printf("[DEBUG] create lru cache, maxKeys=%d, maxValueSize=%d, maxCacheSize=%d",
-		res.maxKeys, res.maxValueSize, res.maxCacheSize)
-	return &res, nil
-}
-
-// Get is loading cache method to get value by key or load via fn if not found
-func (m *memoryCache) Get(key string, fn func() ([]byte, error)) (data []byte, err error) {
-	if b, ok := m.bytesCache.Get(key); ok {
-		return b.([]byte), nil
-	}
-
-	if data, err = fn(); err != nil {
-		return data, err
-	}
-	if m.allowed(data) {
-		m.bytesCache.Add(key, data)
-		atomic.AddInt64(&m.currentSize, int64(len(data)))
-
-		if m.maxCacheSize > 0 && atomic.LoadInt64(&m.currentSize) > m.maxCacheSize {
-			for atomic.LoadInt64(&m.currentSize) > m.maxCacheSize {
-				m.bytesCache.RemoveOldest()
-			}
-		}
-	}
-	return data, nil
-}
-
-// Flush clears cache and calls postFlushFn async
-func (m *memoryCache) Flush(scopes ...string) {
-
-	if len(scopes) == 0 {
-		m.bytesCache.Purge()
-		go m.postFlushFn()
-		return
-	}
-
-	// check if fullKey has matching scopes
-	inScope := func(fullKey string) bool {
-		_, keyScopes, err := ParseKey(fullKey)
-		if err != nil {
-			return false
-		}
-		for _, s := range scopes {
-			for _, ks := range keyScopes {
-				if ks == s {
-					return true
-				}
-			}
-		}
-		return false
-	}
-
-	for _, k := range m.bytesCache.Keys() {
-		key := k.(string)
-		if inScope(key) {
-			m.bytesCache.Remove(key) // Keys() returns copy of cache's key, safe to remove directly
-		}
-	}
-
-	if m.postFlushFn != nil {
-		go m.postFlushFn()
-	}
-}
-
-func (m *memoryCache) allowed(data []byte) bool {
-	if m.maxValueSize > 0 && len(data) >= m.maxValueSize {
-		return false
-	}
-	return true
-}

app/rest/cache/memory_test.go

@@ -1,310 +0,0 @@
-package cache
-
-import (
-	"fmt"
-	"math/rand"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMemoryCache_Get(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }))
-	require.Nil(t, err)
-	res, err := lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	res, err = lc.Get("key", func() ([]byte, error) {
-		atomic.AddInt32(&coldCalls, 1)
-		return []byte("result"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result", string(res))
-	assert.Equal(t, int32(1), atomic.LoadInt32(&coldCalls))
-	assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-
-	lc.Flush()
-	time.Sleep(100 * time.Millisecond) // let postFn to do its thing
-	assert.Equal(t, int32(1), atomic.LoadInt32(&postFnCall))
-
-	_, err = lc.Get("key", func() ([]byte, error) {
-		return nil, errors.New("err")
-	})
-	assert.NotNil(t, err)
-}
-
-func TestMemoryCache_MaxKeys(t *testing.T) {
-	var postFnCall, coldCalls int32
-	lc, err := NewMemoryCache(PostFlushFn(func() { atomic.AddInt32(&postFnCall, 1) }),
-		MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-
-	// put 5 keys to cache
-	for i := 0; i < 5; i++ {
-		res, e := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-			atomic.AddInt32(&coldCalls, 1)
-			return []byte(fmt.Sprintf("result-%d", i)), nil
-		})
-		assert.Nil(t, e)
-		assert.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-		assert.Equal(t, int32(i+1), atomic.LoadInt32(&coldCalls))
-		assert.Equal(t, int32(0), atomic.LoadInt32(&postFnCall))
-	}
-
-	// check if really cached
-	res, err := lc.Get("key-3", func() ([]byte, error) {
-		return []byte("result-blah"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-3", string(res), "should be cached")
-
-	// try to cache after maxKeys reached
-	res, err = lc.Get("key-X", func() ([]byte, error) {
-		return []byte("result-X"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-X", string(res))
-
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-
-	// put to cache and make sure it cached
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-	assert.Equal(t, 5, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxValueSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(5), MaxValSize(10))
-	require.Nil(t, err)
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-
-	res, err = lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Zzzz"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res), "got cached value")
-
-	// put too big value to cache and make sure it is not cached
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("1234567890"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "1234567890", string(res))
-
-	res, err = lc.Get("key-Big", func() ([]byte, error) {
-		return []byte("result-big"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-big", string(res), "got not cached value")
-}
-
-func TestMemoryCache_MaxCacheSize(t *testing.T) {
-	lc, err := NewMemoryCache(MaxKeys(50), MaxCacheSize(20))
-	require.Nil(t, err)
-
-	// put good size value to cache and make sure it cached
-	res, err := lc.Get("key-Z", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "result-Z", string(res))
-	assert.Equal(t, int64(8), lc.(*memoryCache).currentSize)
-
-	_, err = lc.Get("key-Z2", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	// this will cause removal
-	_, err = lc.Get("key-Z3", func() ([]byte, error) {
-		return []byte("result-Z"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, int64(16), lc.(*memoryCache).currentSize)
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_MaxCacheSizeParallel(t *testing.T) {
-	lc, err := NewMemoryCache(MaxCacheSize(123), MaxKeys(10000))
-	require.Nil(t, err)
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Nanosecond)
-			defer wg.Done()
-			res, err := lc.Get(fmt.Sprintf("key-%d", i), func() ([]byte, error) {
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, fmt.Sprintf("result-%d", i), string(res))
-			size := atomic.LoadInt64(&lc.(*memoryCache).currentSize)
-			require.True(t, size < 200 && size >= 0, "unexpected size=%d", size) // won't be exactly 123 due parallel
-		}()
-	}
-	wg.Wait()
-	assert.True(t, lc.(*memoryCache).currentSize < 123 && lc.(*memoryCache).currentSize >= 0)
-	t.Log("size=", lc.(*memoryCache).currentSize)
-}
-
-func TestMemoryCache_Parallel(t *testing.T) {
-	var coldCalls int32
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get("key", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	wg := sync.WaitGroup{}
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			defer wg.Done()
-			res, err := lc.Get("key", func() ([]byte, error) {
-				atomic.AddInt32(&coldCalls, 1)
-				return []byte(fmt.Sprintf("result-%d", i)), nil
-			})
-			require.Nil(t, err)
-			require.Equal(t, "value", string(res))
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, int32(0), atomic.LoadInt32(&coldCalls))
-}
-
-func TestMemoryCache_Scopes(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	res, err := lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(res))
-
-	res, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		return []byte("value2"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value2", string(res))
-
-	assert.Equal(t, 2, lc.(*memoryCache).bytesCache.Len())
-	lc.Flush("s1")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	_, err = lc.Get(Key("key2", "s2"), func() ([]byte, error) {
-		assert.Fail(t, "should stay")
-		return nil, nil
-	})
-	assert.Nil(t, err)
-	res, err = lc.Get(Key("key", "s1", "s2"), func() ([]byte, error) {
-		return []byte("value-upd"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value-upd", string(res), "was deleted, update")
-}
-
-func TestMemoryCache_Flush(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-
-	addToCache := func(key string, scopes ...string) {
-		res, err := lc.Get(key, func() ([]byte, error) {
-			return []byte("value" + key), nil
-		})
-		require.Nil(t, err)
-		require.Equal(t, "value"+key, string(res))
-	}
-
-	init := func() {
-		lc.Flush()
-		addToCache(Key("key1", "s1", "s2"))
-		addToCache(Key("key2", "s1", "s2", "s3"))
-		addToCache(Key("key3", "s1", "s2", "s3"))
-		addToCache(Key("key4", "s2", "s3"))
-		addToCache(Key("key5", "s2"))
-		addToCache(Key("key6"))
-		addToCache(Key("key7", "s4", "s3"))
-		require.Equal(t, 7, lc.(*memoryCache).bytesCache.Len(), "cache init")
-	}
-
-	tbl := []struct {
-		scopes []string
-		left   int
-		msg    string
-	}{
-		{[]string{}, 0, "full flush, no scopes"},
-		{[]string{"s0"}, 7, "flush wrong scope"},
-		{[]string{"s1"}, 4, "flush s1 scope"},
-		{[]string{"s2", "s1"}, 2, "flush s2+s1 scope"},
-		{[]string{"s1", "s2"}, 2, "flush s1+s2 scope"},
-		{[]string{"s1", "s2", "s4"}, 1, "flush s1+s2+s4 scope"},
-		{[]string{"s1", "s2", "s3"}, 1, "flush s1+s2+s3 scope"},
-		{[]string{"s1", "s2", "ss"}, 2, "flush s1+s2+wrong scope"},
-	}
-
-	for i, tt := range tbl {
-		init()
-		lc.Flush(tt.scopes...)
-		assert.Equal(t, tt.left, lc.(*memoryCache).bytesCache.Len(), "keys size, %s #%d", tt.msg, i)
-	}
-}
-
-func TestMemoryCache_FlushFailed(t *testing.T) {
-	lc, err := NewMemoryCache()
-	require.Nil(t, err)
-	val, err := lc.Get("invalid-composite", func() ([]byte, error) {
-		return []byte("value"), nil
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, "value", string(val))
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-
-	lc.Flush("invalid-composite")
-	assert.Equal(t, 1, lc.(*memoryCache).bytesCache.Len())
-}
-
-func TestMemoryCache_BadOptions(t *testing.T) {
-	_, err := NewMemoryCache(MaxCacheSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size or MaxCacheSize, -1")
-
-	_, err = NewMemoryCache(MaxKeys(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxKeys, -1")
-
-	_, err = NewMemoryCache(MaxValSize(-1))
-	assert.EqualError(t, err, "failed to set cache option: negative size for MaxValSize, -1")
-}

app/rest/cache/options.go

@@ -1,50 +0,0 @@
-package cache
-
-import "github.com/pkg/errors"
-
-// Option func type
-type Option func(lc *memoryCache) error
-
-// MaxValSize functional option defines the largest value's size allowed to be cached
-// By default it is 0, which means unlimited.
-func MaxValSize(max int) Option {
-	return func(lc *memoryCache) error {
-		lc.maxValueSize = max
-		if max <= 0 {
-			return errors.Errorf("negative size for MaxValSize, %d", max)
-		}
-		return nil
-	}
-}
-
-// MaxKeys functional option defines how many keys to keep.
-// By default it is 0, which means unlimited.
-func MaxKeys(max int) Option {
-	return func(lc *memoryCache) error {
-		lc.maxKeys = max
-		if max <= 0 {
-			return errors.Errorf("negative size for MaxKeys, %d", max)
-		}
-		return nil
-	}
-}
-
-// MaxCacheSize functional option defines the total size of cached data.
-// By default it is 0, which means unlimited.
-func MaxCacheSize(max int64) Option {
-	return func(lc *memoryCache) error {
-		lc.maxCacheSize = max
-		if max <= 0 {
-			return errors.Errorf("negative size or MaxCacheSize, %d", max)
-		}
-		return nil
-	}
-}
-
-// PostFlushFn functional option defines how callback function called after each Flush.
-func PostFlushFn(postFlushFn func()) Option {
-	return func(lc *memoryCache) error {
-		lc.postFlushFn = postFlushFn
-		return nil
-	}
-}

app/rest/httperrors.go

@@ -1,44 +0,0 @@
-package rest
-
-import (
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"runtime"
-	"strings"
-
-	"github.com/go-chi/render"
-)
-
-// SendErrorJSON makes {error: blah, details: blah} json body and responds with error code
-func SendErrorJSON(w http.ResponseWriter, r *http.Request, code int, err error, details string) {
-	log.Printf("[DEBUG] %s", errDetailsMsg(r, code, err, details))
-	render.Status(r, code)
-	render.JSON(w, r, map[string]interface{}{"error": err.Error(), "details": details})
-}
-
-func errDetailsMsg(r *http.Request, code int, err error, details string) string {
-	uinfoStr := ""
-	if user, e := GetUserInfo(r); e == nil {
-		uinfoStr = user.Name + "/" + user.ID + " - "
-	}
-	q := r.URL.String()
-	if qun, e := url.QueryUnescape(q); e == nil {
-		q = qun
-	}
-
-	srcFileInfo := ""
-	if pc, file, line, ok := runtime.Caller(2); ok {
-		fnameElems := strings.Split(file, "/")
-		funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-		srcFileInfo = fmt.Sprintf(" [caused by %s:%d %s]", strings.Join(fnameElems[len(fnameElems)-3:], "/"),
-			line, funcNameElems[len(funcNameElems)-1])
-	}
-
-	remoteIP := r.RemoteAddr
-	if pos := strings.Index(remoteIP, ":"); pos >= 0 {
-		remoteIP = remoteIP[:pos]
-	}
-	return fmt.Sprintf("%s - %v - %d - %s%s - %s%s", details, err, code, uinfoStr, remoteIP, q, srcFileInfo)
-}

app/rest/httperrors_test.go

@@ -1,60 +0,0 @@
-package rest
-
-import (
-	"errors"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestSendErrorJSON(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/error" {
-			t.Log("http err request", r.URL)
-			SendErrorJSON(w, r, 500, errors.New("error 500"), "error details 123456")
-			return
-		}
-		w.WriteHeader(404)
-	}))
-
-	defer ts.Close()
-
-	resp, err := http.Get(ts.URL + "/error")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
-	assert.Equal(t, 500, resp.StatusCode)
-
-	assert.Equal(t, `{"details":"error details 123456","error":"error 500"}`+"\n", string(body))
-}
-
-func TestErrorDetailsMsg(t *testing.T) {
-	callerFn := func() {
-		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		require.Nil(t, err)
-		req.RemoteAddr = "1.2.3.4"
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - 1.2.3.4 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:47 rest.TestErrorDetailsMsg]", msg)
-	}
-	callerFn()
-}
-
-func TestErrorDetailsMsgWithUser(t *testing.T) {
-	callerFn := func() {
-		req, err := http.NewRequest("GET", "https://example.com/test?k1=v1&k2=v2", nil)
-		req.RemoteAddr = "127.0.0.1:1234"
-		req = SetUserInfo(req, store.User{Name: "test", ID: "id"})
-		require.Nil(t, err)
-		msg := errDetailsMsg(req, 500, errors.New("error 500"), "error details 123456")
-		assert.Equal(t, "error details 123456 - error 500 - 500 - test/id - 127.0.0.1 - https://example.com/test?k1=v1&k2=v2 [caused by app/rest/httperrors_test.go:59 rest.TestErrorDetailsMsgWithUser]", msg)
-	}
-	callerFn()
-}

app/rest/proxy/avatar.go

@@ -1,119 +0,0 @@
-package proxy
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/rest"
-	"github.com/umputun/remark/app/store"
-)
-
-// Avatar provides file-system store and http handler for avatars
-// On user login auth will call Put and it will retrieve and save picture locally.
-type Avatar struct {
-	Store     AvatarStore
-	RoutePath string
-	RemarkURL string
-}
-
-const imgSfx = ".image"
-
-// Put stores retrieved avatar to StorePath. Gets image from user info. Returns proxied url
-func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
-
-	// no picture for user, try default avatar
-	if u.Picture == "" {
-		return "", errors.Errorf("no picture for %s", u.ID)
-	}
-
-	// load avatar from remote location
-	client := http.Client{Timeout: 10 * time.Second}
-	var resp *http.Response
-	err = retry(5, time.Second, func() error {
-		var e error
-		resp, e = client.Get(u.Picture)
-		return e
-	})
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch avatar from the orig")
-	}
-
-	defer func() {
-		if e := resp.Body.Close(); e != nil {
-			log.Printf("[WARN] can't close response body, %s", e)
-		}
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", errors.Errorf("failed to get avatar from the orig, status %s", resp.Status)
-	}
-
-	avatar, err := p.Store.Put(u.ID, resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	log.Printf("[DEBUG] saved avatar from %s to %s, user %q", u.Picture, avatar, u.Name)
-	return p.RemarkURL + p.RoutePath + "/" + avatar, nil
-}
-
-// Routes returns auth routes for given provider
-func (p *Avatar) Routes(middlewares ...func(http.Handler) http.Handler) (string, chi.Router) {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-
-	// GET /123456789.image
-	router.Get("/{avatar}", func(w http.ResponseWriter, r *http.Request) {
-
-		avatar := chi.URLParam(r, "avatar")
-
-		// enforce client-side caching
-		etag := `"` + avatar + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-
-		avReader, size, err := p.Store.Get(avatar)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load avatar")
-			return
-		}
-
-		defer func() {
-			if e := avReader.Close(); e != nil {
-				log.Printf("[WARN] can't close avatar reader for %s, %s", avatar, e)
-			}
-		}()
-
-		w.Header().Set("Content-Type", "image/*")
-		w.Header().Set("Content-Length", strconv.Itoa(size))
-		w.WriteHeader(http.StatusOK)
-		if _, err = io.Copy(w, avReader); err != nil {
-			log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
-		}
-	})
-
-	return p.RoutePath, router
-}
-
-func retry(retries int, delay time.Duration, fn func() error) (err error) {
-	for i := 0; i < retries; i++ {
-		if err = fn(); err == nil {
-			return nil
-		}
-		time.Sleep(delay)
-	}
-	return errors.Wrap(err, "retry failed")
-}

app/rest/proxy/avatar_store.go

@@ -1,145 +0,0 @@
-package proxy
-
-//go:generate sh -c "mockery -inpkg -name AvatarStore -print > /tmp/mock.tmp && mv /tmp/mock.tmp avatar_store_mock.go"
-
-import (
-	"bytes"
-	"fmt"
-	"hash/crc64"
-	"image"
-	"image/png"
-	"io"
-	"log"
-	"os"
-	"path"
-	"strings"
-	"sync"
-
-	// Initializing packages for supporting GIF and JPEG formats.
-	_ "image/gif"
-	_ "image/jpeg"
-
-	"github.com/pkg/errors"
-	"golang.org/x/image/draw"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// AvatarStore defines interface to store and serve avatars
-type AvatarStore interface {
-	Put(userID string, reader io.Reader) (avatar string, err error)
-	Get(avatar string) (reader io.ReadCloser, size int, err error)
-}
-
-// FSAvatarStore implements AvatarStore for local file system
-type FSAvatarStore struct {
-	storePath   string
-	resizeLimit int
-	ctcTable    *crc64.Table
-	once        sync.Once
-}
-
-// NewFSAvatarStore makes file-system avatar store
-func NewFSAvatarStore(storePath string, resizeLimit int) *FSAvatarStore {
-	return &FSAvatarStore{storePath: storePath, resizeLimit: resizeLimit}
-}
-
-// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
-func (fs *FSAvatarStore) Put(userID string, reader io.Reader) (avatar string, err error) {
-	id := store.EncodeID(userID)
-	location := fs.location(id) // location adds partition to path
-
-	if _, err = os.Stat(location); os.IsNotExist(err) {
-		if e := os.Mkdir(location, 0700); e != nil {
-			return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
-		}
-	}
-
-	avFile := path.Join(location, id+imgSfx)
-	fh, err := os.Create(avFile)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't create file %s", avFile)
-	}
-	defer func() {
-		if e := fh.Close(); e != nil {
-			log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
-		}
-	}()
-
-	// Trying to resize avatar.
-	if reader = resize(reader, fs.resizeLimit); reader == nil {
-		return "", errors.New("avatar reader is nil")
-	}
-
-	if _, err = io.Copy(fh, reader); err != nil {
-		return "", errors.Wrapf(err, "can't save file %s", avFile)
-	}
-	return id + imgSfx, nil
-}
-
-// Get avatar reader for avatar id.image
-func (fs *FSAvatarStore) Get(avatar string) (reader io.ReadCloser, size int, err error) {
-	location := fs.location(strings.TrimSuffix(avatar, imgSfx))
-	avFile := path.Join(location, avatar)
-	fh, err := os.Open(avFile)
-	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't load avatar %s, id")
-	}
-	if fi, e := fh.Stat(); e == nil {
-		size = int(fi.Size())
-	}
-	return fh, size, nil
-}
-
-// get location (directory) for user id by adding partition to final path in order to keep files
-// in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/avatars.test/92
-func (fs *FSAvatarStore) location(id string) string {
-	fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
-	checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
-	partition := checksum64 % 100
-	return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
-}
-
-// Resizes an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
-// (width or height) preserving aspect ratio.
-// Returns original reader if resizing is not needed or failed.
-func resize(reader io.Reader, limit int) io.Reader {
-	if reader == nil {
-		log.Print("[WARN] avatar resize(): reader is nil")
-		return nil
-	}
-	if limit <= 0 {
-		log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
-		return reader
-	}
-
-	var teeBuf bytes.Buffer
-	tee := io.TeeReader(reader, &teeBuf)
-	src, _, err := image.Decode(tee)
-	if err != nil {
-		log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
-		return &teeBuf
-	}
-
-	bounds := src.Bounds()
-	w, h := bounds.Dx(), bounds.Dy()
-	if w <= limit && h <= limit || w <= 0 || h <= 0 {
-		log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
-		return &teeBuf
-	}
-	newW, newH := w*limit/h, limit
-	if w > h {
-		newW, newH = limit, h*limit/w
-	}
-	m := image.NewRGBA(image.Rect(0, 0, newW, newH))
-	// Slower than `draw.ApproxBiLinear.Scale()` but better quality.
-	draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
-
-	var out bytes.Buffer
-	if err = png.Encode(&out, m); err != nil {
-		log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
-		return &teeBuf
-	}
-	return &out
-}

app/rest/proxy/avatar_store_test.go

@@ -1,135 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"image"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestAvatarStore_Put(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	avatar, err := p.Put("user1", nil)
-	assert.Equal(t, "", avatar)
-	assert.EqualError(t, err, "avatar reader is nil")
-
-	avatar, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	require.Nil(t, err)
-	assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	avatar, err = p.Put("user2", strings.NewReader("some picture bin data 123"))
-	require.Nil(t, err)
-	assert.Equal(t, "a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(25), fi.Size())
-
-	// with resize
-	file, e := os.Open("testdata/circles.png")
-	require.Nil(t, e)
-	avatar, err = p.Put("user3", file)
-	require.Nil(t, err)
-	assert.Equal(t, "0b7f849446d3383546d15a480966084442cd2193.image", avatar)
-	fi, err = os.Stat("/tmp/avatars.test/60/0b7f849446d3383546d15a480966084442cd2193.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(6986), fi.Size())
-
-	p = NewFSAvatarStore("/dev/null", 300)
-	_, err = p.Put("user1", strings.NewReader("some picture bin data"))
-	assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
-}
-
-func TestAvatarStore_Get(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-	os.MkdirAll("/tmp/avatars.test/30", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-	err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
-	assert.Nil(t, err)
-	r, size, err := p.Get("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.Nil(t, err)
-	assert.Equal(t, 9, size)
-	data, err := ioutil.ReadAll(r)
-	assert.Nil(t, err)
-	assert.Equal(t, "something", string(data))
-}
-
-func TestAvatarStore_Location(t *testing.T) {
-	p := NewFSAvatarStore("/tmp/avatars.test", 300)
-
-	tbl := []struct {
-		id  string
-		res string
-	}{
-		{"abc", "/tmp/avatars.test/35"},
-		{"xyz", "/tmp/avatars.test/69"},
-		{"blah blah", "/tmp/avatars.test/29"},
-	}
-
-	for i, tt := range tbl {
-		assert.Equal(t, tt.res, p.location(tt.id), "test #%d", i)
-	}
-}
-
-func TestAvatarStore_resize(t *testing.T) {
-	checkC := func(t *testing.T, r io.Reader, cExp []byte) {
-		content, err := ioutil.ReadAll(r)
-		require.NoError(t, err)
-		assert.Equal(t, cExp, content)
-	}
-
-	// Reader is nil.
-	resizedR := resize(nil, 100)
-	// assert.EqualError(t, err, "limit should be greater than 0")
-	assert.Nil(t, resizedR)
-
-	// Negative limit error.
-	resizedR = resize(strings.NewReader("some picture bin data"), -1)
-	require.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("some picture bin data"))
-
-	// Decode error.
-	resizedR = resize(strings.NewReader("invalid image content"), 100)
-	assert.NotNil(t, resizedR)
-	checkC(t, resizedR, []byte("invalid image content"))
-
-	cases := []struct {
-		file   string
-		wr, hr int
-	}{
-		{"testdata/circles.png", 400, 300}, // full size: 800x600 px
-		{"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
-	}
-
-	for _, c := range cases {
-		img, err := ioutil.ReadFile(c.file)
-		require.Nil(t, err, "can't open test file %s", c.file)
-
-		// No need for resize, avatar dimensions are smaller than resize limit.
-		resizedR = resize(bytes.NewReader(img), 800)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-		checkC(t, resizedR, img)
-
-		// Resizing to half of width. Check resizedR avatar format PNG.
-		resizedR = resize(bytes.NewReader(img), 400)
-		assert.NotNilf(t, resizedR, "file %s", c.file)
-
-		imgRz, format, err := image.Decode(resizedR)
-		assert.Nilf(t, err, "file %s", c.file)
-		assert.Equalf(t, "png", format, "file %s", c.file)
-		bounds := imgRz.Bounds()
-		assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
-		assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
-	}
-}

app/rest/proxy/avatar_test.go

@@ -1,142 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestAvatar_Put(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	res, err := p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/b3daa77b4c04a9551b8781d03191fe098f325e67.image", res)
-	fi, err := os.Stat("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-
-	u.ID = "user2"
-	res, err = p.Put(u)
-	assert.NoError(t, err)
-	assert.Equal(t, "http://localhost:8080/avatar/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image", res)
-	fi, err = os.Stat("/tmp/avatars.test/84/a1881c06eec96db9901c7bbfe41c42a3f08e9cb4.image")
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), fi.Size())
-}
-
-func TestAvatar_PutFailed(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		log.Print("request: ", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-
-	u := store.User{ID: "user1", Name: "user1 name"}
-	_, err := p.Put(u)
-	assert.EqualError(t, err, "no picture for user1")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: "http://127.0.0.1:12345/avater/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "connect: connection refused")
-
-	u = store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/avatar/pic"}
-	_, err = p.Put(u)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "failed to get avatar from the orig")
-}
-
-func TestAvatar_Routes(t *testing.T) {
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/pic.png" {
-			w.Header().Set("Content-Type", "image/*")
-			w.Header().Set("Custom-Header", "xyz")
-			fmt.Fprint(w, "some picture bin data")
-			return
-		}
-		http.Error(w, "not found", http.StatusNotFound)
-	}))
-	defer ts.Close()
-
-	p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
-	os.MkdirAll("/tmp/avatars.test", 0700)
-	defer os.RemoveAll("/tmp/avatars.test")
-
-	u := store.User{ID: "user1", Name: "user1 name", Picture: ts.URL + "/pic.png"}
-	_, err := p.Put(u)
-	assert.NoError(t, err)
-
-	req, err := http.NewRequest("GET", "/b3daa77b4c04a9551b8781d03191fe098f325e67.image", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	rr := httptest.NewRecorder()
-	_, routes := p.Routes()
-	handler := http.Handler(routes)
-	handler.ServeHTTP(rr, req)
-
-	assert.Equal(t, http.StatusOK, rr.Code)
-
-	assert.Equal(t, []string{"image/*"}, rr.HeaderMap["Content-Type"])
-	assert.Equal(t, []string{"21"}, rr.HeaderMap["Content-Length"])
-	assert.Equal(t, []string(nil), rr.HeaderMap["Custom-Header"], "strip all custom headers")
-	assert.NotNil(t, rr.HeaderMap["Etag"])
-
-	bb := bytes.Buffer{}
-	sz, err := io.Copy(&bb, rr.Body)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(21), sz)
-	assert.Equal(t, "some picture bin data", bb.String())
-}
-
-func TestAvatar_Retry(t *testing.T) {
-	i := 0
-	err := retry(5, time.Millisecond, func() error {
-		if i == 3 {
-			return nil
-		}
-		i++
-		return errors.New("err")
-	})
-	assert.Nil(t, err)
-	assert.Equal(t, 3, i)
-
-	st := time.Now()
-	err = retry(5, time.Millisecond, func() error {
-		return errors.New("err")
-	})
-	assert.NotNil(t, err)
-	assert.True(t, time.Since(st) >= time.Microsecond*5)
-}

app/rest/proxy/image.go

@@ -1,126 +0,0 @@
-package proxy
-
-import (
-	"encoding/base64"
-	"io"
-	"log"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
-	"github.com/pkg/errors"
-	"github.com/umputun/remark/app/rest"
-)
-
-// Image extracts image src from comment's html and provides proxy for them
-// this is needed to keep remark42 running behind of HTTPS serve all images via https
-type Image struct {
-	RemarkURL string
-	RoutePath string
-	Enabled   bool
-}
-
-// Convert all img src links without https to proxied links
-func (p Image) Convert(commentHTML string) string {
-	if !p.Enabled || strings.HasPrefix(p.RemarkURL, "http://") {
-		return commentHTML
-	}
-
-	imgs, err := p.extract(commentHTML)
-	if err != nil {
-		return commentHTML
-	}
-
-	return p.replace(commentHTML, imgs)
-}
-
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
-	if !p.Enabled {
-		return router
-	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url")
-			return
-		}
-
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = retry(5, time.Second, func() error {
-			var e error
-			resp, e = client.Get(string(src))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src))
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
-
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
-
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
-		}
-	})
-	return router
-}
-
-// extract gets all non-https images and return list of src
-func (p Image) extract(commentHTML string) ([]string, error) {
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
-	if err != nil {
-		return nil, errors.Wrap(err, "can't create document")
-	}
-	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
-		if im, ok := s.Attr("src"); ok {
-			if strings.HasPrefix(im, "http://") {
-				result = append(result, im)
-			}
-		}
-	})
-	return result, nil
-}
-
-// replace img links in commentHTML with route to proxy with base64 encoded original link
-func (p Image) replace(commentHTML string, imgs []string) string {
-
-	for _, img := range imgs {
-		encodedImgURL := base64.URLEncoding.EncodeToString([]byte(img))
-		resImgURL := p.RemarkURL + p.RoutePath + "?src=" + encodedImgURL
-		commentHTML = strings.Replace(commentHTML, img, resImgURL, -1)
-	}
-
-	return commentHTML
-}

app/rest/proxy/image_test.go

@@ -1,121 +0,0 @@
-package proxy
-
-import (
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestPicture_Extract(t *testing.T) {
-
-	tbl := []struct {
-		inp string
-		res []string
-	}{
-		{
-			`<p> blah <img src="http://radio-t.com/img.png"/> test</p>`,
-			[]string{"http://radio-t.com/img.png"},
-		},
-		{
-			`<p> blah <img src="https://radio-t.com/img.png"/> test</p>`,
-			[]string{},
-		},
-		{
-			`<img src="http://radio-t.com/img2.png"/>`,
-			[]string{"http://radio-t.com/img2.png"},
-		},
-		{
-			`<img src="http://radio-t.com/img3.png"/> <div>xyz <img src="http://images.pexels.com/67636/img4.jpeg"> </div>`,
-			[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"},
-		},
-		{
-			`<img src="https://radio-t.com/img3.png"/> <div>xyz <img src="http://images.pexels.com/67636/img4.jpeg"> </div>`,
-			[]string{"http://images.pexels.com/67636/img4.jpeg"},
-		},
-		{
-			`abcd <b>blah</b> <h1>xxx</h1>`,
-			[]string{},
-		},
-	}
-	img := Image{Enabled: true}
-
-	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
-	}
-}
-
-func TestPicture_Replace(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
-	r := img.replace(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`,
-		[]string{"http://radio-t.com/img3.png", "http://images.pexels.com/67636/img4.jpeg"})
-	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-}
-
-func TestImage_Routes(t *testing.T) {
-	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
-
-	httpSrv := imgHTTPServer(t)
-	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
-
-	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	t.Logf("%+v", resp.Header)
-	assert.Equal(t, "123", resp.Header["Content-Length"][0])
-	assert.Equal(t, "image/png", resp.Header["Content-Type"][0])
-
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/no-such-image.png"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 404, resp.StatusCode)
-
-	encodedImgURL = base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "bad encoding"))
-	resp, err = http.Get(ts.URL + "/?src=" + encodedImgURL)
-	require.Nil(t, err)
-	assert.Equal(t, 400, resp.StatusCode)
-}
-
-func TestPicture_Convert(t *testing.T) {
-	img := Image{Enabled: true, RoutePath: "/img"}
-	r := img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
-	assert.Equal(t, `<img src="/img?src=aHR0cDovL3JhZGlvLXQuY29tL2ltZzMucG5n"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-
-	r = img.Convert(`<img src="https://radio-t.com/img3.png"/> xyz <img src="http://images.pexels.com/67636/img4.jpeg">`)
-	assert.Equal(t, `<img src="https://radio-t.com/img3.png"/> xyz <img src="/img?src=aHR0cDovL2ltYWdlcy5wZXhlbHMuY29tLzY3NjM2L2ltZzQuanBlZw==">`, r)
-
-	img = Image{Enabled: true, RoutePath: "/img", RemarkURL: "http://example.com"}
-	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
-	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "http:// remark url, no proxy")
-
-	img = Image{Enabled: false, RoutePath: "/img"}
-	r = img.Convert(`<img src="http://radio-t.com/img3.png"/> xyz`)
-	assert.Equal(t, `<img src="http://radio-t.com/img3.png"/> xyz`, r, "disabled, no proxy")
-}
-
-func imgHTTPServer(t *testing.T) *httptest.Server {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/image/img1.png" {
-			t.Log("http img request", r.URL)
-			w.Header().Add("Content-Length", "123")
-			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
-			return
-		}
-		t.Log("http img request - not found", r.URL)
-		w.WriteHeader(404)
-	}))
-
-	return ts
-}

app/rest/tree.go

@@ -1,155 +0,0 @@
-package rest
-
-import (
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// Tree is formatter making tree from the list of comments
-type Tree struct {
-	Nodes []*Node        `json:"comments"`
-	Info  store.PostInfo `json:"info,omitempty"`
-}
-
-// Node is a comment with optional replies
-type Node struct {
-	Comment    store.Comment `json:"comment"`
-	Replies    []*Node       `json:"replies,omitempty"`
-	tsModified time.Time
-	tsCreated  time.Time
-}
-
-// recurData wraps all fields used in recursive processing as intermediate results
-type recurData struct {
-	tsModified time.Time
-	tsCreated  time.Time
-	visible    bool
-}
-
-// MakeTree gets unsorted list of comments and produces Tree
-// It will make store.PostInfo by itself and will mark Info.ReadOnly based on passed readOnlyAge
-// Tree maker is local and has no access to the data store. By this reason it has to make Info and won't be able
-// to handle store's read-only status. This status should be set by caller.
-func MakeTree(comments []store.Comment, sortType string, readOnlyAge int) *Tree {
-	if len(comments) == 0 {
-		return &Tree{}
-	}
-
-	res := Tree{
-		Info: store.PostInfo{
-			URL:     comments[0].Locator.URL,
-			FirstTS: comments[0].Timestamp,
-			LastTS:  comments[0].Timestamp,
-		},
-	}
-	res.Info.Count = len(res.filter(comments, func(c store.Comment) bool { return !c.Deleted }))
-
-	topComments := res.filter(comments, func(c store.Comment) bool { return c.ParentID == "" })
-
-	res.Nodes = []*Node{}
-	for _, rootComment := range topComments {
-		node := Node{Comment: rootComment}
-
-		rd := recurData{}
-		commentsTree, tsModified, tsCreated := res.proc(comments, &node, &rd, rootComment.ID)
-		// skip deleted with no sub-comments ar all sub-comments deleted
-		if rootComment.Deleted && (len(commentsTree.Replies) == 0 || !rd.visible) {
-			continue
-		}
-
-		commentsTree.tsModified, commentsTree.tsCreated = tsModified, tsCreated
-		if commentsTree.tsCreated.Before(res.Info.FirstTS) {
-			res.Info.FirstTS = commentsTree.tsCreated
-		}
-		if commentsTree.tsModified.After(res.Info.LastTS) {
-			res.Info.LastTS = commentsTree.tsModified
-		}
-
-		res.Info.ReadOnly = readOnlyAge > 0 && !res.Info.FirstTS.IsZero() &&
-			res.Info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-
-		res.Nodes = append(res.Nodes, commentsTree)
-	}
-
-	res.sortNodes(sortType)
-	return &res
-}
-
-// proc makes tree for one top-level comment recursively
-func (t *Tree) proc(comments []store.Comment, node *Node, rd *recurData, parentID string) (*Node, time.Time, time.Time) {
-
-	if rd.tsModified.IsZero() || rd.tsCreated.IsZero() {
-		rd.tsModified, rd.tsCreated = node.Comment.Timestamp, node.Comment.Timestamp
-	}
-
-	repComments := t.filter(comments, func(comment store.Comment) bool { return comment.ParentID == parentID })
-	for _, rc := range repComments {
-		if rc.Timestamp.After(rd.tsModified) {
-			rd.tsModified = rc.Timestamp
-		}
-		if rc.Timestamp.Before(rd.tsCreated) {
-			rd.tsCreated = rc.Timestamp
-		}
-		if !rc.Deleted {
-			rd.visible = true
-		}
-		rnode := &Node{Comment: rc, Replies: []*Node{}}
-		node.Replies = append(node.Replies, rnode)
-		t.proc(comments, rnode, rd, rc.ID)
-	}
-	// replies always sorted by time
-	sort.Slice(node.Replies, func(i, j int) bool {
-		return node.Replies[i].Comment.Timestamp.Before(node.Replies[j].Comment.Timestamp)
-	})
-	return node, rd.tsModified, rd.tsCreated
-}
-
-// filter returns comments for parentID
-func (t *Tree) filter(comments []store.Comment, fn func(comment store.Comment) bool) (f []store.Comment) {
-
-	for _, c := range comments {
-		if fn(c) {
-			f = append(f, c)
-		}
-	}
-	return f
-}
-
-// sort list of nodes, i.e. top-level comments
-// time sort uses tsModified from latest reply
-func (t *Tree) sortNodes(sortType string) {
-
-	sort.Slice(t.Nodes, func(i, j int) bool {
-		switch sortType {
-		case "+time", "-time", "time":
-			if strings.HasPrefix(sortType, "-") {
-				return t.Nodes[i].Comment.Timestamp.After(t.Nodes[j].Comment.Timestamp)
-			}
-			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
-
-		case "+active", "-active", "active":
-			if strings.HasPrefix(sortType, "-") {
-				return t.Nodes[i].tsModified.After(t.Nodes[j].tsModified)
-			}
-			return t.Nodes[i].tsModified.Before(t.Nodes[j].tsModified)
-
-		case "+score", "-score", "score":
-			if strings.HasPrefix(sortType, "-") {
-				if t.Nodes[i].Comment.Score == t.Nodes[j].Comment.Score {
-					return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
-				}
-				return t.Nodes[i].Comment.Score > t.Nodes[j].Comment.Score
-			}
-			if t.Nodes[i].Comment.Score == t.Nodes[j].Comment.Score {
-				return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
-			}
-			return t.Nodes[i].Comment.Score < t.Nodes[j].Comment.Score
-
-		default:
-			return t.Nodes[i].Comment.Timestamp.Before(t.Nodes[j].Comment.Timestamp)
-		}
-	})
-}

app/rest/tree_test.go

@@ -1,396 +0,0 @@
-package rest
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestMakeTree(t *testing.T) {
-
-	loc := store.Locator{URL: "url", SiteID: "site"}
-	ts := func(min int, sec int) time.Time { return time.Date(2017, 12, 25, 19, min, sec, 0, time.UTC) }
-
-	// unsorted by purpose
-	comments := []store.Comment{
-		{Locator: loc, ID: "14", ParentID: "1", Timestamp: ts(46, 14)},
-		{Locator: loc, ID: "1", Timestamp: ts(46, 1)},
-		{Locator: loc, ID: "2", Timestamp: ts(47, 2)},
-		{Locator: loc, ID: "11", ParentID: "1", Timestamp: ts(46, 11)},
-		{Locator: loc, ID: "13", ParentID: "1", Timestamp: ts(46, 13)},
-		{Locator: loc, ID: "12", ParentID: "1", Timestamp: ts(46, 12)},
-		{Locator: loc, ID: "131", ParentID: "13", Timestamp: ts(46, 31)},
-		{Locator: loc, ID: "132", ParentID: "13", Timestamp: ts(46, 32)},
-		{Locator: loc, ID: "21", ParentID: "2", Timestamp: ts(47, 21)},
-		{Locator: loc, ID: "22", ParentID: "2", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "4", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "3", Timestamp: ts(47, 22)},
-		{Locator: loc, ID: "5", Deleted: true},
-		{Locator: loc, ID: "6", Deleted: true},
-		{Locator: loc, ID: "61", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "62", ParentID: "6", Deleted: true},
-		{Locator: loc, ID: "611", ParentID: "61", Deleted: true},
-	}
-
-	res := MakeTree(comments, "time", 0)
-
-	buf := &bytes.Buffer{}
-	enc := json.NewEncoder(buf)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(res)
-	assert.Nil(t, err)
-	expected, actual := cleanFormatting(expJSON, buf.String())
-	assert.Equal(t, expected, actual)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22)}, res.Info)
-
-	res = MakeTree([]store.Comment{}, "time", 0)
-	assert.Equal(t, &Tree{}, res)
-
-	res = MakeTree(comments, "time", 10)
-	assert.Equal(t, store.PostInfo{URL: "url", Count: 12, FirstTS: ts(46, 1), LastTS: ts(47, 22), ReadOnly: true}, res.Info)
-}
-
-func TestTreeSortNodes(t *testing.T) {
-	// unsorted by purpose
-	comments := []store.Comment{
-		{ID: "14", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "132", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 46, 32, 0, time.UTC)},
-		{ID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 1, 0, time.UTC), Score: 2},
-		{ID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 2, 0, time.UTC), Score: 3},
-		{ID: "11", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 11, 0, time.UTC)},
-		{ID: "13", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 13, 0, time.UTC)},
-		{ID: "12", ParentID: "1", Timestamp: time.Date(2017, 12, 25, 19, 46, 14, 0, time.UTC)},
-		{ID: "131", ParentID: "13", Timestamp: time.Date(2017, 12, 25, 19, 50, 31, 0, time.UTC)},
-		{ID: "21", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 21, 0, time.UTC)},
-		{ID: "22", ParentID: "2", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC)},
-		{ID: "4", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 0, time.UTC), Score: -2},
-		{ID: "3", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 100, time.UTC)},
-		{ID: "6", Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 200, time.UTC)},
-		{ID: "5", Deleted: true, Timestamp: time.Date(2017, 12, 25, 19, 47, 22, 150, time.UTC)},
-	}
-
-	res := MakeTree(comments, "+active", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-
-	res = MakeTree(comments, "-active", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "+time", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-time", 0)
-	assert.Equal(t, "6", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "+score", 0)
-	assert.Equal(t, "4", res.Nodes[0].Comment.ID)
-
-	res = MakeTree(comments, "-score", 0)
-	assert.Equal(t, "2", res.Nodes[0].Comment.ID)
-	assert.Equal(t, "1", res.Nodes[1].Comment.ID)
-	assert.Equal(t, "3", res.Nodes[2].Comment.ID)
-	assert.Equal(t, "6", res.Nodes[3].Comment.ID)
-
-	res = MakeTree(comments, "undefined", 0)
-	t.Log(res.Nodes[0].Comment.ID, res.Nodes[0].tsModified)
-	assert.Equal(t, "1", res.Nodes[0].Comment.ID)
-}
-
-func BenchmarkTree(b *testing.B) {
-	comments := []store.Comment{}
-	data, err := ioutil.ReadFile("testfile.json")
-	assert.Nil(b, err)
-	err = json.Unmarshal(data, &comments)
-	assert.Nil(b, err)
-
-	for i := 0; i < b.N; i++ {
-		res := MakeTree(comments, "time", 0)
-		assert.NotNil(b, res)
-	}
-}
-
-const expJSON = `{
-  "comments": [
-    {
-      "comment": {
-        "id": "1",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": { 
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:46:01Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "11",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-          	  "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:11Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "12",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:12Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "13",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:13Z"
-          },
-          "replies": [
-            {
-              "comment": {
-                "id": "131",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:31Z"
-              }
-            },
-            {
-              "comment": {
-                "id": "132",
-                "pid": "13",
-                "text": "",
-                "user": {
-                  "name": "",
-                  "id": "",
-                  "picture": "",
-                  "admin": false
-                },
-                "locator": {
-                  "site": "site",
-                  "url": "url"
-                },
-                "score": 0,
-                "votes": null,
-                "time": "2017-12-25T19:46:32Z"
-              }
-            }
-          ]
-        },
-        {
-          "comment": {
-            "id": "14",
-            "pid": "1",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:46:14Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "2",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:02Z"
-      },
-      "replies": [
-        {
-          "comment": {
-            "id": "21",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:21Z"
-          }
-        },
-        {
-          "comment": {
-            "id": "22",
-            "pid": "2",
-            "text": "",
-            "user": {
-              "name": "",
-              "id": "",
-              "picture": "",
-              "admin": false
-            },
-            "locator": {
-              "site": "site",
-              "url": "url"
-            },
-            "score": 0,
-            "votes": null,
-            "time": "2017-12-25T19:47:22Z"
-          }
-        }
-      ]
-    },
-    {
-      "comment": {
-        "id": "4",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    },
-    {
-      "comment": {
-        "id": "3",
-        "pid": "",
-        "text": "",
-        "user": {
-          "name": "",
-          "id": "",
-          "picture": "",
-          "admin": false
-        },
-        "locator": {
-          "site": "site",
-          "url": "url"
-        },
-        "score": 0,
-        "votes": null,
-        "time": "2017-12-25T19:47:22Z"
-      }
-    }
-  ],
-  "info": {
-	 "url": "url",
-	 "count": 12,
-     "first_time": "2017-12-25T19:46:01Z",
-     "last_time": "2017-12-25T19:47:22Z"
-   }
-}
-`
-
-func cleanFormatting(expected, actual string) (string, string) {
-	reSpaces := regexp.MustCompile(`[\s\p{Zs}]{2,}`)
-
-	expected = strings.Replace(expected, "\n", " ", -1)
-	expected = strings.Replace(expected, "\t", " ", -1)
-	expected = reSpaces.ReplaceAllString(expected, " ")
-
-	actual = strings.Replace(actual, "\n", " ", -1)
-	actual = reSpaces.ReplaceAllString(actual, " ")
-	return expected, actual
-}

app/rest/user.go

@@ -1,32 +0,0 @@
-package rest
-
-import (
-	"context"
-	"errors"
-	"net/http"
-
-	"github.com/umputun/remark/app/store"
-)
-
-type contextKey string
-
-// GetUserInfo returns user from request context
-func GetUserInfo(r *http.Request) (user store.User, err error) {
-
-	ctx := r.Context()
-	if ctx == nil {
-		return store.User{}, errors.New("no info about user")
-	}
-	if u, ok := ctx.Value(contextKey("user")).(store.User); ok {
-		return u, nil
-	}
-
-	return store.User{}, errors.New("user can't be parsed")
-}
-
-// SetUserInfo sets user into request context
-func SetUserInfo(r *http.Request, user store.User) *http.Request {
-	ctx := r.Context()
-	ctx = context.WithValue(ctx, contextKey("user"), user)
-	return r.WithContext(ctx)
-}

app/rest/user_test.go

@@ -1,21 +0,0 @@
-package rest
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/umputun/remark/app/store"
-)
-
-func TestGetUserInfo(t *testing.T) {
-	r, err := http.NewRequest("GET", "http://blah.com", nil)
-	assert.Nil(t, err)
-	_, err = GetUserInfo(r)
-	assert.NotNil(t, err, "no user info")
-
-	r = SetUserInfo(r, store.User{Name: "test", ID: "id"})
-	u, err := GetUserInfo(r)
-	assert.Nil(t, err)
-	assert.Equal(t, store.User{Name: "test", ID: "id"}, u)
-}

app/store/comment.go

@@ -1,103 +0,0 @@
-package store
-
-import (
-	"html/template"
-	"regexp"
-	"time"
-
-	"github.com/microcosm-cc/bluemonday"
-)
-
-// Comment represents a single comment with optional reference to its parent
-type Comment struct {
-	ID        string          `json:"id"`
-	ParentID  string          `json:"pid"`
-	Text      string          `json:"text"`
-	Orig      string          `json:"orig,omitempty"`
-	User      User            `json:"user"`
-	Locator   Locator         `json:"locator"`
-	Score     int             `json:"score"`
-	Votes     map[string]bool `json:"votes"`
-	Timestamp time.Time       `json:"time"`
-	Edit      *Edit           `json:"edit,omitempty"` // pointer to have empty default in json response
-	Pin       bool            `json:"pin,omitempty"`
-	Deleted   bool            `json:"delete,omitempty"`
-}
-
-// Locator keeps site and url of the post
-type Locator struct {
-	SiteID string `json:"site,omitempty"`
-	URL    string `json:"url"`
-}
-
-// Edit indication
-type Edit struct {
-	Timestamp time.Time `json:"time"`
-	Summary   string    `json:"summary"`
-}
-
-// PostInfo holds summary for given post url
-type PostInfo struct {
-	URL      string    `json:"url"`
-	Count    int       `json:"count"`
-	ReadOnly bool      `json:"read_only,omitempty"`
-	FirstTS  time.Time `json:"first_time,omitempty"`
-	LastTS   time.Time `json:"last_time,omitempty"`
-}
-
-// BlockedUser holds id and ts for blocked user
-type BlockedUser struct {
-	ID        string    `json:"id"`
-	Name      string    `json:"name"`
-	Timestamp time.Time `json:"time"`
-}
-
-// DeleteMode defines how much comment info will be erased
-type DeleteMode int
-
-// DeleteMode enum
-const (
-	SoftDelete DeleteMode = 0
-	HardDelete DeleteMode = 1
-)
-
-// PrepareUntrusted pre-processes a comment received from untrusted source by clearing all
-// autogen fields and reset everything users not supposed to provide
-func (c *Comment) PrepareUntrusted() {
-	c.ID = ""                 // don't allow user to define ID, force auto-gen
-	c.Timestamp = time.Time{} // reset time, force auto-gen
-	c.Votes = make(map[string]bool)
-	c.Score = 0
-	c.Edit = nil
-	c.Pin = false
-	c.Deleted = false
-}
-
-// SetDeleted clears comment info, reset to deleted state. hard flag will clear all user info as well
-func (c *Comment) SetDeleted(mode DeleteMode) {
-	c.Text = ""
-	c.Orig = ""
-	c.Score = 0
-	c.Votes = map[string]bool{}
-	c.Edit = nil
-	c.Deleted = true
-	c.Pin = false
-
-	if mode == HardDelete {
-		c.User.Name = "deleted"
-		c.User.ID = "deleted"
-		c.User.Picture = ""
-		c.User.IP = ""
-	}
-}
-
-// Sanitize clean dangerous html/js from the comment
-func (c *Comment) Sanitize() {
-	p := bluemonday.UGCPolicy()
-	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")
-	c.Text = p.Sanitize(c.Text)
-	c.Orig = p.Sanitize(c.Orig)
-	c.User.ID = template.HTMLEscapeString(c.User.ID)
-	c.User.Name = template.HTMLEscapeString(c.User.Name)
-	c.User.Picture = p.Sanitize(c.User.Picture)
-}

app/store/comment_test.go

@@ -1,112 +0,0 @@
-package store
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestComment_Sanitize(t *testing.T) {
-
-	tbl := []struct {
-		inp Comment
-		out Comment
-	}{
-		{inp: Comment{}, out: Comment{}},
-		{
-			inp: Comment{
-				Text: `blah <a href="javascript:alert('XSS1')" onmouseover="alert('XSS2')">XSS<a>` + "\n\t",
-				User: User{ID: `<a href="http://blah.com">username</a>`},
-			},
-			out: Comment{
-				Text: "blah XSS\n\t",
-				User: User{ID: `&lt;a href=&#34;http://blah.com&#34;&gt;username&lt;/a&gt;`},
-			},
-		},
-	}
-
-	for n, tt := range tbl {
-		tt.inp.Sanitize()
-		assert.Equal(t, tt.out, tt.inp, "check #%d", n)
-	}
-}
-
-func TestComment_PrepareUntrusted(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "username"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Pin:       true,
-		Deleted:   true,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-	}
-
-	comment.PrepareUntrusted()
-	assert.Equal(t, "", comment.ID)
-	assert.Equal(t, "p123", comment.ParentID)
-	assert.Equal(t, "blah", comment.Text)
-	assert.Equal(t, 0, comment.Score)
-	assert.Equal(t, false, comment.Pin)
-	assert.Equal(t, time.Time{}, comment.Timestamp)
-	assert.Equal(t, false, comment.Deleted)
-	assert.Equal(t, make(map[string]bool), comment.Votes)
-	assert.Equal(t, User{ID: "username"}, comment.User)
-
-}
-
-func TestComment_SetDeleted(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Deleted:   false,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-		Pin:       true,
-	}
-
-	comment.SetDeleted(SoftDelete)
-
-	assert.Equal(t, "", comment.Text)
-	assert.Equal(t, "", comment.Orig)
-	assert.Equal(t, map[string]bool{}, comment.Votes)
-	assert.Equal(t, 0, comment.Score)
-	assert.True(t, comment.Deleted)
-	assert.Nil(t, comment.Edit)
-	assert.False(t, comment.Pin)
-	assert.Equal(t, User{Name: "username", ID: "userid", Picture: "pic", Admin: false, Blocked: false, IP: "123"}, comment.User)
-}
-
-func TestComment_SetDeletedHard(t *testing.T) {
-	comment := Comment{
-		Text:      `blah`,
-		User:      User{ID: "userid", Name: "username", IP: "123", Picture: "pic"},
-		ParentID:  "p123",
-		ID:        "123",
-		Locator:   Locator{SiteID: "site", URL: "url"},
-		Score:     10,
-		Deleted:   false,
-		Timestamp: time.Date(2018, 1, 1, 9, 30, 0, 0, time.Local),
-		Votes:     map[string]bool{"uu": true},
-		Pin:       true,
-	}
-
-	comment.SetDeleted(HardDelete)
-
-	assert.Equal(t, "", comment.Text)
-	assert.Equal(t, "", comment.Orig)
-	assert.Equal(t, map[string]bool{}, comment.Votes)
-	assert.Equal(t, 0, comment.Score)
-	assert.True(t, comment.Deleted)
-	assert.Nil(t, comment.Edit)
-	assert.False(t, comment.Pin)
-	assert.Equal(t, User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comment.User)
-}

app/store/engine/bolt_accessor.go

@@ -1,530 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// BoltDB implements store.Interface, represents multiple sites with multiplexing to different bolt dbs. Thread safe.
-// there are 5 types of top-level buckets:
-//  - comments for post in "posts" top-level bucket. Each url (post) makes its own bucket and each k:v pair is commentID:comment
-//  - history of all comments. They all in a single "last" bucket (per site) and key is defined by ref struct as ts+commentID
-//    value is not full comment but a reference combined from post-url+commentID
-//  - user to comment references in "users" bucket. It used to get comments for user. Key is userID and value
-//    is a nested bucket named userID with kv as ts:reference
-//  - blocking info sits in "block" bucket. Key is userID, value - ts
-//  - counts per post to keep number of comments. Key is post url, value - count
-//  - readonly per post to keep status of manually set RO posts. Key is post url, value - ts
-type BoltDB struct {
-	dbs map[string]*bolt.DB
-}
-
-const (
-	// top level buckets
-	postsBucketName    = "posts"
-	lastBucketName     = "last"
-	userBucketName     = "users"
-	blocksBucketName   = "block"
-	infoBucketName     = "info"
-	readonlyBucketName = "readonly"
-	verifiedBucketName = "verified"
-
-	// limits
-	lastLimit = 1000
-	userLimit = 500
-)
-
-const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-
-// BoltSite defines single site param
-type BoltSite struct {
-	FileName string // full path to boltdb
-	SiteID   string // ID to access given site
-}
-
-// NewBoltDB makes persistent boltdb-based store
-func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
-	log.Printf("[INFO] bolt store for sites %+v", sites)
-	result := BoltDB{dbs: make(map[string]*bolt.DB)}
-	for _, site := range sites {
-		db, err := bolt.Open(site.FileName, 0600, &options) // bolt.Options{Timeout: 30 * time.Second}
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to make boltdb for %s", site.FileName)
-		}
-
-		// make top-level buckets
-		topBuckets := []string{postsBucketName, lastBucketName, userBucketName, blocksBucketName,
-			infoBucketName, readonlyBucketName, verifiedBucketName}
-		err = db.Update(func(tx *bolt.Tx) error {
-			for _, bktName := range topBuckets {
-				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-					return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-				}
-			}
-			return nil
-		})
-
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create top level bucket)")
-		}
-
-		result.dbs[site.SiteID] = db
-	}
-	return &result, nil
-}
-
-// Create saves new comment to store. Adds to posts bucket, reference to last and user bucket and increments count bucket
-func (b *BoltDB) Create(comment store.Comment) (commentID string, err error) {
-
-	bdb, err := b.db(comment.Locator.SiteID)
-	if err != nil {
-		return "", err
-	}
-
-	if b.IsReadOnly(comment.Locator) {
-		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
-	}
-
-	err = bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.makePostBucket(tx, comment.Locator.URL)
-		if e != nil {
-			return e
-		}
-
-		// check if key already in store, reject doubles
-		if postBkt.Get([]byte(comment.ID)) != nil {
-			return errors.Errorf("key %s already in store", comment.ID)
-		}
-
-		// serialize comment to json []byte for bolt and save
-		if e = b.save(postBkt, []byte(comment.ID), comment); e != nil {
-			return errors.Wrapf(e, "failed to put key %s to bucket %s", comment.ID, comment.Locator.URL)
-		}
-
-		ref := b.makeRef(comment)
-
-		// add reference to comment to "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		commentTs := []byte(comment.Timestamp.Format(tsNano))
-		e = lastBkt.Put(commentTs, ref)
-		if e != nil {
-			return errors.Wrapf(e, "can't put reference %s to %s", ref, lastBucketName)
-		}
-
-		// add reference to commentID to "users" bucket
-		userBkt, e := b.getUserBucket(tx, comment.User.ID)
-		if e != nil {
-			return errors.Wrapf(e, "can't get bucket %s", comment.User.ID)
-		}
-		// put into individual user's bucket with ts as a key
-		if e = userBkt.Put(commentTs, ref); e != nil {
-			return errors.Wrapf(e, "failed to put user comment %s for %s", comment.ID, comment.User.ID)
-		}
-
-		// set info with countfor post url
-		if _, e = b.setInfo(tx, comment); e != nil {
-			return errors.Wrapf(e, "failed to set info for %s", comment.Locator)
-		}
-		return nil
-	})
-
-	return comment.ID, err
-}
-
-// Find returns all comments for post and sorts results
-func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
-	comments = []store.Comment{}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		return bucket.ForEach(func(k, v []byte) error {
-			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
-				return errors.Wrap(e, "failed to unmarshal")
-			}
-			comments = append(comments, comment)
-			return nil
-		})
-	})
-
-	comments = sortComments(comments, sortFld)
-	return comments, err
-}
-
-// Last returns up to max last comments for given siteID
-func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err error) {
-
-	if max > lastLimit || max == 0 {
-		max = lastLimit
-	}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		c := lastBkt.Cursor()
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			url, commentID, e := b.parseRef(v)
-			if e != nil {
-				return e
-			}
-			postBkt, e := b.getPostBucket(tx, url)
-			if e != nil {
-				return e
-			}
-
-			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
-				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
-				continue
-			}
-			if comment.Deleted {
-				continue
-			}
-			comments = append(comments, comment)
-			if len(comments) >= max {
-				break
-			}
-		}
-		return nil
-	})
-
-	return comments, err
-}
-
-// Count returns number of comments for locator
-func (b *BoltDB) Count(locator store.Locator) (count int, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return 0, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		var e error
-		count, e = b.count(tx, locator.URL, 0)
-		return e
-	})
-
-	return count, err
-}
-
-// List returns list of all commented posts with counters
-// uses count bucket to get number of comments
-func (b BoltDB) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		postsBkt := tx.Bucket([]byte(postsBucketName))
-
-		c := postsBkt.Cursor()
-		n := 0
-		for k, _ := c.Last(); k != nil; k, _ = c.Prev() {
-			n++
-			if skip > 0 && n <= skip {
-				continue
-			}
-			postURL := string(k)
-			infoBkt := tx.Bucket([]byte(infoBucketName))
-			info := store.PostInfo{}
-			if e := b.load(infoBkt, []byte(postURL), &info); e != nil {
-				return errors.Wrapf(e, "can't load info for %s", postURL)
-			}
-			list = append(list, info)
-			if limit > 0 && len(list) >= limit {
-				break
-			}
-		}
-		return nil
-	})
-
-	return list, err
-}
-
-// Info returns time range and count for locator
-func (b *BoltDB) Info(locator store.Locator, readOnlyAge int) (store.PostInfo, error) {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return store.PostInfo{}, err
-	}
-
-	info := store.PostInfo{}
-	err = bdb.View(func(tx *bolt.Tx) error {
-		infoBkt := tx.Bucket([]byte(infoBucketName))
-		if e := b.load(infoBkt, []byte(locator.URL), &info); e != nil {
-			return errors.Wrapf(e, "can't load info for %s", locator.URL)
-		}
-		return nil
-	})
-
-	// set read-only from age and manual bucket
-	info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
-	if b.IsReadOnly(locator) {
-		info.ReadOnly = true
-	}
-	return info, err
-}
-
-// User extracts all comments for given site and given userID
-// "users" bucket has sub-bucket for each userID, and keeps it as ts:ref
-func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
-
-	comments = []store.Comment{}
-	commentRefs := []string{}
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	if limit == 0 || limit > userLimit {
-		limit = userLimit
-	}
-
-	// get list of references to comments
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-
-		c := userIDBkt.Cursor()
-		skipComments := 0
-		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			if len(commentRefs) >= limit {
-				break
-			}
-			if skip > 0 && skipComments < skip {
-				skipComments++
-				continue
-			}
-			commentRefs = append(commentRefs, string(v))
-		}
-		return nil
-	})
-
-	if err != nil {
-		return comments, err
-	}
-
-	// retrieve comments for refs
-	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
-		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
-			comments = append(comments, c)
-		}
-	}
-
-	return comments, err
-}
-
-// UserCount returns number of comments for user
-func (b *BoltDB) UserCount(siteID, userID string) (int, error) {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return 0, err
-	}
-	count := 0
-	err = bdb.View(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		userIDBkt := usersBkt.Bucket([]byte(userID))
-		if userIDBkt == nil {
-			return errors.Errorf("no comments for user %s in store", userID)
-		}
-		stats := userIDBkt.Stats()
-		count = stats.KeyN
-		return nil
-	})
-	return count, err
-}
-
-// Get returns comment for locator.URL and commentID string
-func (b *BoltDB) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return comment, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.load(bucket, []byte(commentID), &comment)
-	})
-	return comment, err
-}
-
-// Put updates comment for locator.URL with mutable part of comment
-func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
-
-	if curComment, err := b.Get(locator, comment.ID); err == nil {
-		// preserve immutable fields
-		comment.ParentID = curComment.ParentID
-		comment.Locator = curComment.Locator
-		comment.Timestamp = curComment.Timestamp
-		comment.User = curComment.User
-	}
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-		return b.save(bucket, []byte(comment.ID), comment)
-	})
-}
-
-// getPostBucket return bucket with all comments for postURL
-func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res := postsBkt.Bucket([]byte(postURL))
-	if res == nil {
-		return nil, errors.Errorf("no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-// makePostBucket create new bucket for postURL as a key. This bucket holds all comments for the post.
-func (b *BoltDB) makePostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
-	postsBkt := tx.Bucket([]byte(postsBucketName))
-	if postsBkt == nil {
-		return nil, errors.Errorf("no bucket %s", postsBucketName)
-	}
-	res, err := postsBkt.CreateBucketIfNotExists([]byte(postURL))
-	if err != nil {
-		return nil, errors.Wrapf(err, "no bucket %s in store", postURL)
-	}
-	return res, nil
-}
-
-func (b *BoltDB) getUserBucket(tx *bolt.Tx, userID string) (*bolt.Bucket, error) {
-	usersBkt := tx.Bucket([]byte(userBucketName))
-	userIDBkt, e := usersBkt.CreateBucketIfNotExists([]byte(userID)) // get bucket for userID
-	if e != nil {
-		return nil, errors.Wrapf(e, "can't get bucket %s", userID)
-	}
-	return userIDBkt, nil
-}
-
-// save marshaled value to key for bucket. Should run in update tx
-func (b *BoltDB) save(bkt *bolt.Bucket, key []byte, value interface{}) (err error) {
-	if value == nil {
-		return errors.Errorf("can't save nil value for %s", key)
-	}
-	jdata, jerr := json.Marshal(value)
-	if jerr != nil {
-		return errors.Wrap(jerr, "can't marshal comment")
-	}
-	if err = bkt.Put(key, jdata); err != nil {
-		return errors.Wrapf(err, "failed to save key %s", key)
-	}
-	return nil
-}
-
-// load and unmarshal json value by key from bucket. Should run in view tx
-func (b *BoltDB) load(bkt *bolt.Bucket, key []byte, res interface{}) error {
-	value := bkt.Get(key)
-	if value == nil {
-		return errors.Errorf("no value for %s", key)
-	}
-
-	if err := json.Unmarshal(value, &res); err != nil {
-		return errors.Wrap(err, "failed to unmarshal")
-	}
-	return nil
-}
-
-// count adds val to counts key postURL. val can be negative to subtract. if val 0 can be used as accessor
-// it uses separate counts bucket because boltdb Stat call is very slow
-func (b *BoltDB) count(tx *bolt.Tx, postURL string, val int) (int, error) {
-
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(postURL), &info); err != nil {
-		info = store.PostInfo{}
-	}
-	if val == 0 { // get current count, don't update
-		return info.Count, nil
-	}
-	info.Count += val
-
-	return info.Count, b.save(infoBkt, []byte(postURL), &info)
-}
-
-func (b *BoltDB) setInfo(tx *bolt.Tx, comment store.Comment) (store.PostInfo, error) {
-	infoBkt := tx.Bucket([]byte(infoBucketName))
-	info := store.PostInfo{}
-	if err := b.load(infoBkt, []byte(comment.Locator.URL), &info); err != nil {
-		info = store.PostInfo{
-			Count:   0,
-			URL:     comment.Locator.URL,
-			FirstTS: comment.Timestamp,
-			LastTS:  comment.Timestamp,
-		}
-	}
-	info.Count++
-	info.LastTS = comment.Timestamp
-	return info, b.save(infoBkt, []byte(comment.Locator.URL), &info)
-}
-
-func (b *BoltDB) db(siteID string) (*bolt.DB, error) {
-	if res, ok := b.dbs[siteID]; ok {
-		return res, nil
-	}
-	return nil, errors.Errorf("site %q not found", siteID)
-}
-
-// makeRef creates reference combining url and comment id
-func (b *BoltDB) makeRef(comment store.Comment) []byte {
-	return []byte(fmt.Sprintf("%s!!%s", comment.Locator.URL, comment.ID))
-}
-
-// parseRef gets parts of reference
-func (b *BoltDB) parseRef(val []byte) (url string, id string, err error) {
-	elems := strings.Split(string(val), "!!")
-	if len(elems) != 2 {
-		return "", "", errors.Errorf("invalid reference value %s", string(val))
-	}
-	return elems[0], elems[1], nil
-}

app/store/engine/bolt_accessor_test.go

@@ -1,372 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-var testDb = "test-remark.db"
-
-func TestBoltDB_CreateAndFind(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
-	assert.Equal(t, "user1", res[0].User.ID)
-	t.Log(res[0].ID)
-
-	_, err = b.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
-	assert.NotNil(t, err)
-	assert.Equal(t, "key id-1 already in store", err.Error())
-
-	_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltDB_CreateReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	var b = prep(t)
-
-	comment := store.Comment{
-		ID:        "id-ro",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	err := b.SetReadOnly(comment.Locator, true)
-	require.Nil(t, err)
-
-	_, err = b.Create(comment)
-	assert.NotNil(t, err)
-	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
-
-	err = b.SetReadOnly(comment.Locator, false)
-	require.Nil(t, err)
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-}
-
-func TestBoltDB_Get(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "some text2", comment.Text)
-
-	comment, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
-	assert.NotNil(t, err)
-
-	_, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Put(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-
-	comment := res[0]
-	comment.Text = "abc 123"
-	comment.Score = 100
-	err = b.Put(loc, comment)
-	assert.Nil(t, err)
-
-	comment, err = b.Get(loc, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "abc 123", comment.Text)
-	assert.Equal(t, res[0].ID, comment.ID)
-	assert.Equal(t, 100, comment.Score)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	err = b.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
-	assert.EqualError(t, err, `no bucket https://radio-t.com-bad in store`)
-}
-
-func TestBoltDB_Last(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	res, err = b.Last("radio-t", 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "some text2", res[0].Text)
-
-	_, err = b.Last("bad", 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Count(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 2, c)
-
-	c, err = b.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c)
-
-	_, err = b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "bad"})
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_List(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local) }
-
-	res, err := b.List("radio-t", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
-		res)
-
-	res, err = b.List("radio-t", -1, -1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
-		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	res, err = b.List("radio-t", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
-
-	res, err = b.List("radio-t", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
-
-	_, err = b.List("bad", 1, 1)
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltDB_Info(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t) // two comments for https://radio-t.com
-
-	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local) }
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	r, err := b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
-	require.NotNil(t, err)
-
-	_, err = b.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
-	require.NotNil(t, err)
-
-	err = b.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
-	require.Nil(t, err)
-	r, err = b.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
-	require.Nil(t, err)
-	assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
-
-}
-
-func TestBoltDB_GetForUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	res, err := b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
-
-	res, err = b.User("radio-t", "user1", 1, 1)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(res), "allow 1 comment")
-	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
-
-	_, err = b.User("bad", "user1", 1, 0)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.User("radio-t", "userZ", 1, 0)
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_GetForUserPagination(t *testing.T) {
-	os.Remove(testDb)
-	b, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	defer os.Remove(testDb)
-
-	c := store.Comment{
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:    store.User{ID: "user1", Name: "user name"},
-	}
-
-	// write 200 comments
-	for i := 0; i < 200; i++ {
-		c.ID = fmt.Sprintf("id-%d", i)
-		c.Text = fmt.Sprintf("text #%d", i)
-		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
-		_, err = b.Create(c)
-		require.Nil(t, err)
-	}
-
-	// get all comments
-	res, err := b.User("radio-t", "user1", 0, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 200, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-
-	// seek 0, 5 comments
-	res, err = b.User("radio-t", "user1", 5, 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-199", res[0].ID)
-	assert.Equal(t, "id-195", res[4].ID)
-
-	// seek 10, 3 comments
-	res, err = b.User("radio-t", "user1", 3, 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, "id-189", res[0].ID)
-	assert.Equal(t, "id-187", res[2].ID)
-
-	// seek 195, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 195)
-	assert.Nil(t, err)
-	assert.Equal(t, 5, len(res))
-	assert.Equal(t, "id-4", res[0].ID)
-	assert.Equal(t, "id-0", res[4].ID)
-
-	// seek 255, ask 10 comments
-	res, err = b.User("radio-t", "user1", 10, 255)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(res))
-}
-
-func TestBoltDB_GetForUserCounter(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	count, err := b.UserCount("radio-t", "user1")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, count)
-
-	_, err = b.UserCount("bad", "user1")
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	_, err = b.UserCount("radio-t", "userZ")
-	assert.EqualError(t, err, `no comments for user userZ in store`)
-}
-
-func TestBoltDB_Ref(t *testing.T) {
-	b := BoltDB{}
-	comment := store.Comment{
-		ID:        "12345",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	res := b.makeRef(comment)
-	assert.Equal(t, "https://radio-t.com/2!!12345", string(res))
-
-	url, id, err := b.parseRef([]byte("https://radio-t.com/2!!12345"))
-	assert.Nil(t, err)
-	assert.Equal(t, "https://radio-t.com/2", url)
-	assert.Equal(t, "12345", id)
-
-	_, _, err = b.parseRef([]byte("https://radio-t.com/2"))
-	assert.NotNil(t, err)
-}
-
-func TestBoltDB_New(t *testing.T) {
-	_, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: "/tmp/no-such-place/tmp.db", SiteID: "radio-t"})
-	assert.EqualError(t, err, "failed to make boltdb for /tmp/no-such-place/tmp.db: open /tmp/no-such-place/tmp.db: no such file or directory")
-}
-
-// makes new boltdb, put two records
-func prep(t *testing.T) *BoltDB {
-	os.Remove(testDb)
-
-	boltStore, err := NewBoltDB(bolt.Options{}, BoltSite{FileName: testDb, SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b := boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/store/engine/bolt_admin.go

@@ -1,305 +0,0 @@
-package engine
-
-import (
-	"encoding/json"
-	"log"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-)
-
-// Delete removes comment, by locator from the store.
-// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
-// From last bucket removed for real.
-func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-
-		postBkt, e := b.getPostBucket(tx, locator.URL)
-		if e != nil {
-			return e
-		}
-
-		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
-			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
-		}
-		// set deleted status and clear fields
-		comment.SetDeleted(mode)
-
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
-			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
-		}
-
-		// delete from "last" bucket
-		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
-			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
-		}
-
-		// decrement comments count for post url
-		if _, e = b.count(tx, comment.Locator.URL, -1); e != nil {
-			return errors.Wrapf(e, "failed to decrement count for %s", comment.Locator)
-		}
-
-		return nil
-	})
-}
-
-// DeleteAll removes all top-level buckets for given siteID
-func (b *BoltDB) DeleteAll(siteID string) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// delete all buckets except blocked users
-	toDelete := []string{postsBucketName, lastBucketName, userBucketName, infoBucketName}
-
-	// delete top-level buckets
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		for _, bktName := range toDelete {
-
-			if e := tx.DeleteBucket([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to delete top level bucket %s", bktName)
-			}
-			if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
-				return errors.Wrapf(err, "failed to create top level bucket %s", bktName)
-			}
-		}
-		return nil
-	})
-
-	return errors.Wrapf(err, "failed to delete top level buckets fro site %s", siteID)
-}
-
-// DeleteUser removes all comments for given user. Everything will be market as deleted
-// and user name and userID will be changed to "deleted". Also removes from last and from user buckets.
-func (b *BoltDB) DeleteUser(siteID string, userID string) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	// get list of all comments outside of transaction loop
-	posts, err := b.List(siteID, 0, 0)
-	if err != nil {
-		return err
-	}
-
-	type commentInfo struct {
-		locator   store.Locator
-		commentID string
-	}
-
-	// get list of commentID for all user's comment
-	comments := []commentInfo{}
-	for _, postInfo := range posts {
-		err = bdb.View(func(tx *bolt.Tx) error {
-			postsBkt := tx.Bucket([]byte(postsBucketName))
-			postBkt := postsBkt.Bucket([]byte(postInfo.URL))
-			err = postBkt.ForEach(func(postURL []byte, commentVal []byte) error {
-				comment := store.Comment{}
-				if err = json.Unmarshal(commentVal, &comment); err != nil {
-					return errors.Wrap(err, "failed to unmarshal")
-				}
-				if comment.User.ID == userID {
-					comments = append(comments, commentInfo{locator: comment.Locator, commentID: comment.ID})
-				}
-				return nil
-			})
-			return errors.Wrapf(err, "failed to collect list of comments for deletion from %s", postInfo.URL)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Printf("[DEBUG] comments for removal=%d", len(comments))
-
-	// delete collected comments
-	for _, ci := range comments {
-		if e := b.Delete(ci.locator, ci.commentID, store.HardDelete); e != nil {
-			return errors.Wrapf(err, "failed to delete comment %+v", ci)
-		}
-	}
-
-	//  delete  user bucket
-	err = bdb.Update(func(tx *bolt.Tx) error {
-		usersBkt := tx.Bucket([]byte(userBucketName))
-		if usersBkt != nil {
-			if e := usersBkt.DeleteBucket([]byte(userID)); e != nil {
-				return errors.Wrapf(err, "failed to delete user bucket for %s", userID)
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "can't delete user meta")
-	}
-
-	if len(comments) == 0 {
-		return errors.Errorf("unknown user %s", userID)
-	}
-
-	return err
-}
-
-// SetBlock blocks/unblocks user for given site
-func (b *BoltDB) SetBlock(siteID string, userID string, status bool) error {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to put %s to %s", userID, blocksBucketName)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean %s from %s", userID, blocksBucketName)
-			}
-		}
-		return nil
-	})
-}
-
-// IsBlocked checks if user blocked
-func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		blocked = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return blocked
-}
-
-// Blocked get lists of blocked users for given site
-// bucket uses userID:
-func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
-	users = []store.BlockedUser{}
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return nil, err
-	}
-
-	err = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(blocksBucketName))
-		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
-			}
-
-			// get user name from comment user section
-			userName := ""
-			userComments, e := b.User(siteID, string(k), 1, 0)
-			if e == nil && len(userComments) > 0 {
-				userName = userComments[0].User.Name
-			}
-			users = append(users, store.BlockedUser{ID: string(k), Name: userName, Timestamp: ts})
-			return nil
-		})
-	})
-
-	return users, err
-}
-
-// SetReadOnly makes post read-only or reset the ro flag
-func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(locator.URL), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set ro for %s", locator.URL)
-			}
-		case false:
-			if e := bucket.Delete([]byte(locator.URL)); e != nil {
-				return errors.Wrapf(e, "failed to clean ro for %s", locator.URL)
-			}
-		}
-		return nil
-	})
-}
-
-// IsReadOnly checks if user blocked
-func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
-
-	bdb, err := b.db(locator.SiteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(readonlyBucketName))
-		ro = bucket.Get([]byte(locator.URL)) != nil
-		return nil
-	})
-	return ro
-}
-
-// SetVerified makes user verified or reset the flag
-func (b *BoltDB) SetVerified(siteID string, userID string, status bool) error {
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return err
-	}
-
-	return bdb.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		switch status {
-		case true:
-			if e := bucket.Put([]byte(userID), []byte(time.Now().Format(tsNano))); e != nil {
-				return errors.Wrapf(e, "failed to set verified status for %s", userID)
-			}
-		case false:
-			if e := bucket.Delete([]byte(userID)); e != nil {
-				return errors.Wrapf(e, "failed to clean verified status for %s", userID)
-			}
-		}
-		return nil
-	})
-}
-
-// IsVerified checks if user verified
-func (b *BoltDB) IsVerified(siteID string, userID string) (verified bool) {
-
-	bdb, err := b.db(siteID)
-	if err != nil {
-		return false
-	}
-
-	_ = bdb.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(verifiedBucketName))
-		verified = bucket.Get([]byte(userID)) != nil
-		return nil
-	})
-	return verified
-}

app/store/engine/bolt_admin_test.go

@@ -1,200 +0,0 @@
-package engine
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestBoltAdmin_Delete(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-
-	assert.Equal(t, "some text2", res[1].Text)
-	assert.False(t, res[1].Deleted)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
-
-	err = b.Delete(loc, "123456", store.SoftDelete)
-	assert.NotNil(t, err)
-
-	loc.SiteID = "bad"
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `site "bad" not found`)
-
-	loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
-	err = b.Delete(loc, res[0].ID, store.SoftDelete)
-	assert.EqualError(t, err, `no bucket https://radio-t.com/bad in store`)
-}
-
-func TestBoltAdmin_DeleteHard(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.Delete(loc, res[0].ID, store.HardDelete)
-	assert.Nil(t, err)
-
-	res, err = b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, "", res[0].Text)
-	assert.True(t, res[0].Deleted, "marked deleted")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-}
-
-func TestBoltAdmin_DeleteAll(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "initially 2 comments")
-
-	err = b.DeleteAll("radio-t")
-	assert.Nil(t, err)
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	err = b.DeleteAll("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_DeleteUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-	err := b.DeleteUser("radio-t", "user1")
-	require.NoError(t, err)
-
-	loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
-	res, err := b.Find(loc, "time")
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res), "2 comments with deleted info")
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
-
-	c, err := b.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	assert.Equal(t, 0, c, "0 count")
-
-	_, err = b.User("radio-t", "user1", 5, 0)
-	assert.EqualError(t, err, "no comments for user user1 in store")
-
-	comments, err := b.Last("radio-t", 10)
-	assert.Nil(t, err)
-	assert.Equal(t, 0, len(comments), "nothing left")
-
-	err = b.DeleteUser("radio-t-bad", "user1")
-	assert.EqualError(t, err, `site "radio-t-bad" not found`)
-}
-
-func TestBoltAdmin_BlockUser(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "nothing blocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true))
-	assert.True(t, b.IsBlocked("radio-t", "user1"), "user1 blocked")
-
-	assert.False(t, b.IsBlocked("radio-t", "user2"), "user2 still unblocked")
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", false))
-	assert.False(t, b.IsBlocked("radio-t", "user1"), "user1 unblocked")
-
-	assert.EqualError(t, b.SetBlock("bad", "user1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetBlock("radio-t", "userX", false))
-
-	assert.False(t, b.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_BlockList(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.NoError(t, b.SetBlock("radio-t", "user1", true))
-	assert.NoError(t, b.SetBlock("radio-t", "user2", true))
-	assert.NoError(t, b.SetBlock("radio-t", "user3", false))
-
-	ids, err := b.Blocked("radio-t")
-	assert.NoError(t, err)
-
-	assert.Equal(t, 2, len(ids))
-	assert.Equal(t, "user1", ids[0].ID)
-	assert.Equal(t, "user2", ids[1].ID)
-	t.Logf("%+v", ids)
-
-	_, err = b.Blocked("bad")
-	assert.EqualError(t, err, `site "bad" not found`)
-}
-
-func TestBoltAdmin_ReadOnly(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
-	assert.True(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
-
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
-
-	assert.EqualError(t, b.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), `site "bad" not found`)
-	assert.NoError(t, b.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
-
-	assert.False(t, b.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
-}
-
-func TestBoltAdmin_Verified(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prep(t)
-
-	assert.False(t, b.IsVerified("radio-t", "u1"), "nothing verified")
-
-	assert.NoError(t, b.SetVerified("radio-t", "u1", true))
-	assert.True(t, b.IsVerified("radio-t", "u1"), "u1 verified")
-
-	assert.False(t, b.IsVerified("radio-t", "u2"), "u2 still not verified")
-	assert.NoError(t, b.SetVerified("radio-t", "u1", false))
-	assert.False(t, b.IsVerified("radio-t", "u1"), "u1 not verified anymore")
-
-	assert.EqualError(t, b.SetVerified("bad", "u1", true), `site "bad" not found`)
-	assert.NoError(t, b.SetVerified("radio-t", "u1xyz", false))
-
-	assert.False(t, b.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
-}

app/store/engine/engine.go

@@ -1,83 +0,0 @@
-// Package engine defines interfaces each supported storage should implement.
-// Includes default implementation with boltdb
-package engine
-
-import (
-	"sort"
-	"strings"
-
-	"github.com/umputun/remark/app/store"
-)
-
-//go:generate sh -c "mockery -inpkg -name Interface -print > file.tmp && mv file.tmp engine_mock.go"
-
-// Interface combines all store interfaces
-type Interface interface {
-	Accessor
-	Admin
-}
-
-// UserRequest is the request send to get comments by user
-type UserRequest struct {
-	SiteID string
-	UserID string
-	Limit  int
-	Skip   int
-}
-
-// Accessor defines all usual access ops avail for regular user
-type Accessor interface {
-	Create(comment store.Comment) (commentID string, err error)           // create new comment, avoid dups by id
-	Get(locator store.Locator, commentID string) (store.Comment, error)   // get comment by id
-	Put(locator store.Locator, comment store.Comment) error               // update comment, mutable parts only
-	Find(locator store.Locator, sort string) ([]store.Comment, error)     // find comments for locator
-	Last(siteID string, limit int) ([]store.Comment, error)               // last comments for given site, sorted by time
-	User(siteID, userID string, limit, skip int) ([]store.Comment, error) // comments by user, sorted by time
-	UserCount(siteID, userID string) (int, error)                         // comments count by user
-	Count(locator store.Locator) (int, error)                             // number of comments for the post
-	List(siteID string, limit int, skip int) ([]store.PostInfo, error)    // list of commented posts
-	Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)  // get post info
-}
-
-// Admin defines all store ops avail for admin only
-type Admin interface {
-	Delete(locator store.Locator, commentID string, mode store.DeleteMode) error // delete comment by id
-	DeleteAll(siteID string) error                                               // delete all data from site
-	DeleteUser(siteID string, userID string) error                               // remove all comments from user
-	SetBlock(siteID string, userID string, status bool) error                    // block or unblock  user
-	IsBlocked(siteID string, userID string) bool                                 // check if user blocked
-	Blocked(siteID string) ([]store.BlockedUser, error)                          // get list of blocked users
-	SetReadOnly(locator store.Locator, status bool) error                        // set/reset read-only flag
-	IsReadOnly(locator store.Locator) bool                                       // check if post read-only
-	SetVerified(siteID string, userID string, status bool) error                 // set/reset verified flag
-	IsVerified(siteID string, userID string) bool                                // check verified status
-}
-
-// sortComments is for engines can't sort data internally
-func sortComments(comments []store.Comment, sortFld string) []store.Comment {
-	sort.Slice(comments, func(i, j int) bool {
-		switch sortFld {
-		case "+time", "-time", "time", "+active", "-active", "active":
-			if strings.HasPrefix(sortFld, "-") {
-				return comments[i].Timestamp.After(comments[j].Timestamp)
-			}
-			return comments[i].Timestamp.Before(comments[j].Timestamp)
-
-		case "+score", "-score", "score":
-			if strings.HasPrefix(sortFld, "-") {
-				if comments[i].Score == comments[j].Score {
-					return comments[i].Timestamp.Before(comments[j].Timestamp)
-				}
-				return comments[i].Score > comments[j].Score
-			}
-			if comments[i].Score == comments[j].Score {
-				return comments[i].Timestamp.Before(comments[j].Timestamp)
-			}
-			return comments[i].Score < comments[j].Score
-
-		default:
-			return comments[i].Timestamp.Before(comments[j].Timestamp)
-		}
-	})
-	return comments
-}

app/store/engine/engine_test.go

@@ -1,43 +0,0 @@
-package engine
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/umputun/remark/app/store"
-)
-
-func TestEngine_sortComments(t *testing.T) {
-	cc := []store.Comment{
-		{ID: "1", Score: 5, Timestamp: time.Date(2018, 2, 5, 10, 1, 0, 0, time.Local)},
-		{ID: "2", Score: 4, Timestamp: time.Date(2018, 2, 5, 10, 2, 0, 0, time.Local)},
-		{ID: "3", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 3, 0, 0, time.Local)},
-		{ID: "4", Score: 6, Timestamp: time.Date(2018, 2, 5, 10, 4, 0, 0, time.Local)},
-	}
-
-	sortComments(cc, "+time")
-	assert.Equal(t, "1", cc[0].ID)
-	assert.Equal(t, "2", cc[1].ID)
-	assert.Equal(t, "3", cc[2].ID)
-	assert.Equal(t, "4", cc[3].ID)
-
-	sortComments(cc, "-time")
-	assert.Equal(t, "4", cc[0].ID)
-	assert.Equal(t, "3", cc[1].ID)
-	assert.Equal(t, "2", cc[2].ID)
-	assert.Equal(t, "1", cc[3].ID)
-
-	sortComments(cc, "score")
-	assert.Equal(t, "2", cc[0].ID)
-	assert.Equal(t, "1", cc[1].ID)
-	assert.Equal(t, "3", cc[2].ID)
-	assert.Equal(t, "4", cc[3].ID)
-
-	sortComments(cc, "-score")
-	assert.Equal(t, "3", cc[0].ID)
-	assert.Equal(t, "4", cc[1].ID)
-	assert.Equal(t, "1", cc[2].ID)
-	assert.Equal(t, "2", cc[3].ID)
-}

app/store/service/service.go

@@ -1,189 +0,0 @@
-package service
-
-import (
-	"sync"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-)
-
-// DataStore wraps store.Interface with additional methods
-type DataStore struct {
-	engine.Interface
-	EditDuration   time.Duration
-	Secret         string
-	MaxCommentSize int
-
-	// granular locks
-	scopedLocks struct {
-		sync.Mutex
-		sync.Once
-		locks map[string]sync.Locker
-	}
-}
-
-const defaultCommentMaxSize = 2000
-
-// Create prepares comment and forward to Interface.Create
-func (s *DataStore) Create(comment store.Comment) (commentID string, err error) {
-	// fill ID and time if empty
-	if comment.ID == "" {
-		comment.ID = uuid.New().String()
-	}
-	if comment.Timestamp.IsZero() {
-		comment.Timestamp = time.Now()
-	}
-	// reset votes if nothing
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-
-	comment.Sanitize()            // clear potentially dangerous js from all parts of comment
-	comment.User.HashIP(s.Secret) // replace ip by hash
-
-	return s.Interface.Create(comment)
-}
-
-// SetPin pin/un-pin comment as special
-func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool) error {
-	comment, err := s.Get(locator, commentID)
-	if err != nil {
-		return err
-	}
-	comment.Pin = status
-	return s.Put(locator, comment)
-}
-
-// Vote for comment by id and locator
-func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
-
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
-	defer cLock.Unlock()
-
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	if comment.User.ID == userID && userID != "dev" {
-		return comment, errors.Errorf("user %s can not vote for his own comment %s", userID, commentID)
-	}
-
-	if comment.Votes == nil {
-		comment.Votes = make(map[string]bool)
-	}
-	v, voted := comment.Votes[userID]
-
-	if voted && v == val {
-		return comment, errors.Errorf("user %s already voted for %s", userID, commentID)
-	}
-
-	// reset vote if user changed to opposite
-	if voted && v != val {
-		delete(comment.Votes, userID)
-	}
-
-	// add to voted map if first vote
-	if !voted {
-		comment.Votes[userID] = val
-	}
-
-	// update score
-	if val {
-		comment.Score++
-	} else {
-		comment.Score--
-	}
-
-	return comment, s.Put(locator, comment)
-}
-
-// EditRequest contains fields needed for comment update
-type EditRequest struct {
-	Text    string
-	Orig    string
-	Summary string
-}
-
-// EditComment to edit text and update Edit info
-func (s *DataStore) EditComment(locator store.Locator, commentID string, req EditRequest) (comment store.Comment, err error) {
-	comment, err = s.Get(locator, commentID)
-	if err != nil {
-		return comment, err
-	}
-
-	// edit allowed in editDuration window only
-	if s.EditDuration > 0 && time.Now().After(comment.Timestamp.Add(s.EditDuration)) {
-		return comment, errors.Errorf("too late to edit %s", commentID)
-	}
-
-	comment.Text = req.Text
-	comment.Orig = req.Orig
-	comment.Edit = &store.Edit{
-		Timestamp: time.Now(),
-		Summary:   req.Summary,
-	}
-
-	comment.Sanitize()
-	err = s.Put(locator, comment)
-	return comment, err
-}
-
-// Counts returns postID+count list for given comments
-func (s *DataStore) Counts(siteID string, postIDs []string) ([]store.PostInfo, error) {
-	res := []store.PostInfo{}
-	for _, p := range postIDs {
-		if c, err := s.Count(store.Locator{SiteID: siteID, URL: p}); err == nil {
-			res = append(res, store.PostInfo{URL: p, Count: c})
-		}
-	}
-	return res, nil
-}
-
-// ValidateComment checks if comment size below max and user fields set
-func (s *DataStore) ValidateComment(c *store.Comment) error {
-	maxSize := s.MaxCommentSize
-	if s.MaxCommentSize <= 0 {
-		maxSize = defaultCommentMaxSize
-	}
-	if c.Orig == "" {
-		return errors.New("empty comment text")
-	}
-	if len([]rune(c.Orig)) > maxSize {
-		return errors.Errorf("comment text exceeded max allowed size %d (%d)", maxSize, len([]rune(c.Orig)))
-	}
-	if c.User.ID == "" || c.User.Name == "" {
-		return errors.Errorf("empty user info")
-	}
-	return nil
-}
-
-// IsVerifiedFn returns func to check if user verified or not
-func (s *DataStore) IsVerifiedFn() func(siteID string, userID string) bool {
-	return func(siteID string, userID string) bool {
-		if siteID == "" {
-			return false
-		}
-		return s.IsVerified(siteID, userID)
-	}
-}
-
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
-	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
-
-	s.scopedLocks.Lock()
-	lock, ok := s.scopedLocks.locks[id]
-	if !ok {
-		lock = &sync.Mutex{}
-		s.scopedLocks.locks[id] = lock
-	}
-	s.scopedLocks.Unlock()
-
-	return lock
-}

app/store/service/service_test.go

@@ -1,372 +0,0 @@
-package service
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/coreos/bbolt"
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/app/store"
-	"github.com/umputun/remark/app/store/engine"
-)
-
-var testDb = "/tmp/test-remark.db"
-
-func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	id, err := b.Create(comment)
-	assert.NoError(t, err)
-	assert.True(t, id != "", id)
-
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
-	assert.NoError(t, err)
-	t.Logf("%+v", res)
-	assert.Equal(t, "text", res.Text)
-	assert.True(t, time.Since(res.Timestamp).Seconds() < 1)
-	assert.Equal(t, "user", res.User.ID)
-	assert.Equal(t, "name", res.User.Name)
-	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, map[string]bool{}, res.Votes)
-}
-
-func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), Secret: "secret 123"}
-	comment := store.Comment{
-		Text:      "text",
-		Timestamp: time.Date(2018, 3, 25, 16, 34, 33, 0, time.UTC),
-		Votes:     map[string]bool{"u1": true, "u2": false},
-		User:      store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	id, err := b.Create(comment)
-	assert.NoError(t, err)
-	assert.True(t, id != "", id)
-
-	res, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, id)
-	assert.NoError(t, err)
-	t.Logf("%+v", res)
-	assert.Equal(t, "text", res.Text)
-	assert.Equal(t, comment.Timestamp, res.Timestamp)
-	assert.Equal(t, "user", res.User.ID)
-	assert.Equal(t, "name", res.User.Name)
-	assert.Equal(t, "23f97cf4d5c29ef788ca2bdd1c9e75656c0e4149", res.User.IP)
-	assert.Equal(t, comment.Votes, res.Votes)
-}
-
-func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
-
-	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-	assert.Nil(t, err)
-	assert.Equal(t, 1, c.Score)
-	assert.Equal(t, map[string]bool{"user1": true}, c.Votes, "user voted +")
-
-	c, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user", true)
-	assert.NotNil(t, err, "self-voting not allowed")
-
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-	assert.NotNil(t, err, "double-voting rejected")
-	assert.True(t, strings.HasPrefix(err.Error(), "user user1 already voted"))
-
-	res, err = b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 1, res[0].Score)
-
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", false)
-	assert.Nil(t, err, "vote reset")
-	res, err = b.Last("radio-t", 0)
-	assert.Nil(t, err)
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "vote reset ok")
-}
-
-func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-
-	res, err := b.Last("radio-t", 0)
-	require.Nil(t, err)
-	t.Logf("%+v", res[0])
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 0, res[0].Score)
-	assert.Equal(t, map[string]bool{}, res[0].Votes, "no votes initially")
-
-	// add a vote as user2
-	_, err = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user2", true)
-	require.Nil(t, err)
-
-	// crazy vote +1 as user1
-	var wg sync.WaitGroup
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-
-	t.Logf("%+v", res[0])
-	assert.Equal(t, 3, len(res))
-	assert.Equal(t, 2, res[0].Score, "add single +1")
-	assert.Equal(t, 2, len(res[0].Votes), "made a single vote")
-
-	// random +1/-1 result should be [0..2]
-	rand.Seed(time.Now().UnixNano())
-	for i := 0; i < 1000; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-	assert.Equal(t, 3, len(res))
-	t.Logf("%+v %d", res[0], res[0].Score)
-	assert.True(t, res[0].Score >= 0 && res[0].Score <= 2, "unexpected score %d", res[0].Score)
-}
-
-func TestService_VoteConcurrent(t *testing.T) {
-
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	comment := store.Comment{
-		Text:    "text",
-		User:    store.User{IP: "192.168.1.1", ID: "user", Name: "name"},
-		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-	}
-	_, err := b.Create(comment)
-	assert.NoError(t, err)
-	res, err := b.Last("radio-t", 0)
-	require.Nil(t, err)
-
-	// concurrent vote +1 as multiple users for the same comment
-	var wg sync.WaitGroup
-	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		i := i
-		go func() {
-			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
-		}()
-	}
-	wg.Wait()
-	res, err = b.Last("radio-t", 0)
-	require.NoError(t, err)
-	assert.Equal(t, 100, res[0].Score, "should have 1000 score")
-	assert.Equal(t, 100, len(res[0].Votes), "should have 1000 votes")
-}
-
-func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Equal(t, false, res[0].Pin)
-
-	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, true)
-	assert.Nil(t, err)
-
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, true, c.Pin)
-
-	err = b.SetPin(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, false)
-	assert.Nil(t, err)
-	c, err = b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, false, c.Pin)
-}
-
-func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Nil(t, res[0].Edit)
-
-	comment, err := b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err)
-	assert.Equal(t, "my edit", comment.Edit.Summary)
-	assert.Equal(t, "xxx", comment.Text)
-	assert.Equal(t, "yyy", comment.Orig)
-
-	c, err := b.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID)
-	assert.Nil(t, err)
-	assert.Equal(t, "my edit", c.Edit.Summary)
-	assert.Equal(t, "xxx", c.Text)
-
-	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.Nil(t, err, "allow second edit")
-}
-
-func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond}
-
-	res, err := b.Last("radio-t", 0)
-	t.Logf("%+v", res[0])
-	assert.Nil(t, err)
-	assert.Equal(t, 2, len(res))
-	assert.Nil(t, res[0].Edit)
-
-	time.Sleep(time.Second)
-
-	_, err = b.EditComment(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
-		EditRequest{Orig: "yyy", Text: "xxx", Summary: "my edit"})
-	assert.NotNil(t, err)
-}
-
-func TestService_ValidateComment(t *testing.T) {
-
-	b := DataStore{MaxCommentSize: 2000}
-	longText := fmt.Sprintf("%4000s", "X")
-
-	tbl := []struct {
-		inp store.Comment
-		err error
-	}{
-		{inp: store.Comment{}, err: errors.New("empty comment text")},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid", Name: "name"}}, err: nil},
-		{inp: store.Comment{Orig: "something blah", User: store.User{ID: "myid"}}, err: errors.New("empty user info")},
-		{inp: store.Comment{Orig: longText, User: store.User{ID: "myid", Name: "name"}}, err: errors.New("comment text exceeded max allowed size 2000 (4000)")},
-	}
-
-	for n, tt := range tbl {
-		e := b.ValidateComment(&tt.inp)
-		if tt.err == nil {
-			assert.Nil(t, e, "check #%d", n)
-			continue
-		}
-		assert.EqualError(t, tt.err, e.Error(), "check #%d", n)
-	}
-}
-
-func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
-	b := prepStoreEngine(t) // two comments for https://radio-t.com
-
-	// add one more for https://radio-t.com/2
-	comment := store.Comment{
-		ID:        "123456",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err := b.Create(comment)
-	assert.Nil(t, err)
-
-	svc := DataStore{Interface: b}
-	res, err := svc.Counts("radio-t", []string{"https://radio-t.com/2"})
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1}}, res)
-
-	res, err = svc.Counts("radio-t", []string{"https://radio-t.com", "https://radio-t.com/2", "blah"})
-	assert.Nil(t, err)
-	assert.Equal(t, []store.PostInfo{
-		{URL: "https://radio-t.com", Count: 2},
-		{URL: "https://radio-t.com/2", Count: 1},
-		{URL: "blah", Count: 0},
-	}, res)
-}
-
-func TestService_IsVerifiedFn(t *testing.T) {
-	defer os.Remove(testDb)
-	b := DataStore{Interface: prepStoreEngine(t)}
-
-	fn := b.IsVerifiedFn()
-	verified := fn("radio-t", "user1")
-	assert.False(t, verified)
-
-	err := b.Interface.SetVerified("radio-t", "user1", true)
-	assert.Nil(t, err)
-	verified = fn("radio-t", "user1")
-	assert.True(t, verified)
-}
-
-// makes new boltdb, put two records
-func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
-
-	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
-	assert.Nil(t, err)
-	b := boltStore
-
-	comment := store.Comment{
-		ID:        "id-1",
-		Text:      `some text, <a href="http://radio-t.com">link</a>`,
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	comment = store.Comment{
-		ID:        "id-2",
-		Text:      "some text2",
-		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
-		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
-		User:      store.User{ID: "user1", Name: "user name"},
-	}
-	_, err = b.Create(comment)
-	assert.Nil(t, err)
-
-	return b
-}

app/store/user.go

@@ -1,60 +0,0 @@
-package store
-
-import (
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
-	"hash/crc64"
-	"io"
-	"log"
-	"regexp"
-)
-
-// User holds user-related info
-type User struct {
-	Name     string `json:"name"`
-	ID       string `json:"id"`
-	Picture  string `json:"picture"`
-	IP       string `json:"ip,omitempty"`
-	Admin    bool   `json:"admin"`
-	Blocked  bool   `json:"block,omitempty"`
-	Verified bool   `json:"verified,omitempty"`
-}
-
-var reValidSha = regexp.MustCompile("^[a-fA-F0-9]{40}$")
-
-// HashIP replace IP field with hashed hmac
-func (u *User) HashIP(secret string) {
-	u.IP = HashValue(u.IP, secret)
-}
-
-// HashValue makes hmac with secret
-func HashValue(val string, secret string) string {
-	if val == "" || reValidSha.MatchString(val) {
-		return val // already hashed or empty
-	}
-	key := []byte(secret)
-	h := hmac.New(sha1.New, key)
-	if _, err := io.WriteString(h, val); err != nil {
-		// fail back to crc64
-		log.Printf("[WARN] can't hash ip, %s", err)
-		return fmt.Sprintf("%x", crc64.Checksum([]byte(val), crc64.MakeTable(crc64.ECMA)))
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}
-
-// EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
-// we need hashing for parts of id, in some others hashing for non-User values.
-func EncodeID(id string) string {
-	if reValidSha.MatchString(id) {
-		return id // already hashed or empty
-	}
-	h := sha1.New()
-	if _, err := io.WriteString(h, id); err != nil {
-		// fail back to crc64
-		log.Printf("[WARN] can't hash id %s, %s", id, err)
-		return fmt.Sprintf("%x", crc64.Checksum([]byte(id), crc64.MakeTable(crc64.ECMA)))
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}

app/store/user_test.go

@@ -1,45 +0,0 @@
-package store
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUser_EncodeID(t *testing.T) {
-	tbl := []struct {
-		id   string
-		hash string
-	}{
-		{"myid", "6e34471f84557e1713012d64a7477c71bfdac631"},
-		{"", "da39a3ee5e6b4b0d3255bfef95601890afd80709"},
-		{"blah blah", "135a1e01bae742c4a576b20fd41a683f6483ca43"},
-		{"da39a3ee5e6b4b0d3255bfef95601890afd80709", "da39a3ee5e6b4b0d3255bfef95601890afd80709"},
-	}
-
-	for i, tt := range tbl {
-		assert.Equal(t, tt.hash, EncodeID(tt.id), "case #%d", i)
-	}
-}
-
-func TestUser_HashIP(t *testing.T) {
-	tbl := []struct {
-		ip           string
-		hash1, hash2 string
-	}{
-		{"127.0.0.1", "ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741", "dbc7c999343f003f189f70aaf52cc04443f90790"},
-		{"8.8.8.8", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "70a46afce9633f010b06e129b8ad08243a1c4da9"},
-		{"8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2", "8cee77c27e32a2b5aec95c29888ac9946618d9a2"},
-		{"", "", ""},
-	}
-
-	for i, tt := range tbl {
-		u := User{IP: tt.ip}
-		u.HashIP("")
-		assert.Equal(t, tt.hash1, u.IP, "case #%d", i)
-
-		u = User{IP: tt.ip}
-		u.HashIP("123456")
-		assert.Equal(t, tt.hash2, u.IP, "case #%d", i)
-	}
-}

backend/.golangci.yml

@@ -0,0 +1,74 @@
+run:
+  timeout: 5m
+  output:
+    format: tab
+  skip-dirs:
+    - vendor
+
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0.1
+  maligned:
+    suggest-new: true
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+# TODO: feel free to remove these excludes and fix the code
+      - hugeParam
+      - rangeValCopy
+
+linters:
+  enable:
+    - bodyclose
+    - megacheck
+    - revive
+    - govet
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+    - stylecheck
+    - gochecknoinits
+    - exportloopref
+    - gocritic
+    - nakedret
+    - gosimple
+    - prealloc
+  fast: false
+  disable-all: true
+
+issues:
+  exclude-rules:
+    - text: "at least one file in a package should have a package comment"
+      linters:
+        - stylecheck
+    - text: "should have a package comment, unless it's in another file for this package"
+      linters:
+        - golint
+    - path: _test\.go
+      linters:
+        - gosec
+        - dupl
+  exclude-use-default: false

backend/_example/memory_store/Dockerfile

@@ -0,0 +1,19 @@
+FROM umputun/baseimage:buildgo-latest as build-backend
+
+ADD backend /build/backend
+WORKDIR /build/backend/_example/memory_store
+
+RUN go build -o /build/bin/memory_store -ldflags "-X main.revision=0.0.0 -s -w"
+
+
+FROM umputun/baseimage:app-latest
+
+WORKDIR /srv
+COPY --from=build-backend /build/bin/memory_store /srv/memory_store
+RUN chown -R app:app /srv
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=3s CMD curl --fail http://localhost:8080/ping || exit 1
+USER app
+
+CMD ["/srv/memory_store"]

backend/_example/memory_store/README.md

@@ -0,0 +1,12 @@
+# sample store implementation
+
+`memory_store` illustrates how to make a custom storage plugin for remark42.
+
+In order to run remark42 with memory_store copy provided `compose-dev-memstore.yml` to the root directory and run:
+
+1. `docker-compose -f compose-dev-memstore.yml build`
+1. `docker-compose -f compose-dev-memstore.yml up`
+
+As usual, demo site will run on http://127.0.0.1:8080/web/
+
+note: in order to work with the latest (current) version of master `go.mod` uses replacement directive for the backend package. In real-life usage `replace github.com/umputun/remark42/backend => ../../`  should not be used. 

backend/_example/memory_store/accessor/admin.go

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// MemAdmin implements admin.Store with memory backend
+type MemAdmin struct {
+	data map[string]AdminRec // admin info per site
+	key  string
+}
+
+// AdminRec is a records per site with all admin info in
+type AdminRec struct {
+	SiteID       string
+	IDs          []string // admin ids
+	Email        string   // admin email
+	Enabled      bool     // site enabled
+	CountCreated int64    // number of created posts
+}
+
+// NewMemAdminStore makes admin Store in memory
+func NewMemAdminStore(key string) *MemAdmin {
+	log.Print("[DEBUG] make memory admin store")
+	return &MemAdmin{data: map[string]AdminRec{}, key: key}
+}
+
+// Key executes find by siteID and returns substructure with secret key
+func (m *MemAdmin) Key(_ string) (key string, err error) {
+	return m.key, nil
+}
+
+// Admins executes find by siteID and returns admins ids
+func (m *MemAdmin) Admins(siteID string) (ids []string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return nil, errors.Errorf("site %s not found", siteID)
+	}
+	log.Printf("[DEBUG] admins for %s, %+v", siteID, resp.IDs)
+	return resp.IDs, nil
+}
+
+// Email executes find by siteID and returns admin's email
+func (m *MemAdmin) Email(siteID string) (email string, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return "", errors.Errorf("site %s not found", siteID)
+	}
+
+	return resp.Email, nil
+}
+
+// Enabled return
+func (m *MemAdmin) Enabled(siteID string) (ok bool, err error) {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return false, errors.Errorf("site %s not found", siteID)
+	}
+	return resp.Enabled, nil
+}
+
+// OnEvent reacts on events from updates, created, delete and vote
+func (m *MemAdmin) OnEvent(siteID string, ev admin.EventType) error {
+	resp, ok := m.data[siteID]
+	if !ok {
+		return errors.Errorf("site %s not found", siteID)
+	}
+	if ev == admin.EvCreate {
+		resp.CountCreated++ // not a good idea, just for demo
+		m.data[siteID] = resp
+	}
+	return nil
+}
+
+// Set admin data for siteID
+func (m *MemAdmin) Set(siteID string, arec AdminRec) {
+	m.data[siteID] = arec
+}

backend/_example/memory_store/accessor/admin_test.go

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestMemAdmin_Get(t *testing.T) {
+
+	adm := NewMemAdminStore("secret")
+	var ms admin.Store = adm
+
+	adm.data = map[string]AdminRec{
+		"site1": {"site1", []string{"i11", "i12"}, "e1", true, 0},
+	}
+	adm.Set("site2", AdminRec{"site2", []string{"i21", "i22"}, "e2", true, 0})
+	adm.Set("site3", AdminRec{"site3", []string{"i21", "i22"}, "e3", false, 0})
+
+	admins, err := ms.Admins("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i11", "i12"}, admins)
+	email, err := ms.Email("site1")
+	assert.NoError(t, err)
+	assert.Equal(t, "e1", email)
+	key, err := ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"i21", "i22"}, admins)
+	email, err = ms.Email("site2")
+	assert.NoError(t, err)
+	assert.Equal(t, "e2", email)
+	key, err = ms.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+
+	admins, err = ms.Admins("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, admins)
+
+	email, err = ms.Email("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.Empty(t, email)
+
+	enabled, err := ms.Enabled("site1")
+	assert.NoError(t, err)
+	assert.True(t, enabled)
+
+	enabled, err = ms.Enabled("site3")
+	assert.NoError(t, err)
+	assert.False(t, enabled)
+
+	enabled, err = ms.Enabled("no-site-in-db")
+	assert.EqualError(t, err, "site no-site-in-db not found")
+	assert.False(t, enabled)
+
+	err = ms.OnEvent("site1", admin.EvCreate)
+	assert.NoError(t, err)
+
+	err = ms.OnEvent("no-site-in-db", admin.EvCreate)
+	assert.Error(t, err)
+}

backend/_example/memory_store/accessor/data.go

@@ -0,0 +1,571 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+const lastLimit = 1000
+
+// MemData implements in-memory data store
+type MemData struct {
+	posts     map[string][]store.Comment // key is siteID
+	metaUsers map[string]metaUser        // key is userID
+	metaPosts map[store.Locator]metaPost // key is post's locator
+	mu        sync.RWMutex
+}
+
+type metaPost struct {
+	PostURL  string
+	SiteID   string
+	ReadOnly bool
+}
+
+type metaUser struct {
+	UserID       string
+	SiteID       string
+	Verified     bool
+	Blocked      bool
+	BlockedUntil time.Time
+	Details      engine.UserDetailEntry
+}
+
+// NewMemData makes in-memory engine.
+func NewMemData() *MemData {
+
+	result := &MemData{
+		posts:     map[string][]store.Comment{},
+		metaUsers: map[string]metaUser{},
+		metaPosts: map[store.Locator]metaPost{},
+	}
+	return result
+}
+
+// Create new comment
+func (m *MemData) Create(comment store.Comment) (commentID string, err error) {
+
+	if ro, e := m.Flag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: comment.Locator}); e == nil && ro {
+		return "", errors.Errorf("post %s is read-only", comment.Locator.URL)
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	comments := m.posts[comment.Locator.SiteID]
+	for _, c := range comments { // don't allow duplicated IDs
+		if c.ID == comment.ID {
+			return "", errors.New("dup key")
+		}
+	}
+	comments = append(comments, comment)
+	m.posts[comment.Locator.SiteID] = comments
+	return comment.ID, nil
+}
+
+// Find returns all comments for post and sorts results
+func (m *MemData) Find(req engine.FindRequest) (comments []store.Comment, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	comments = []store.Comment{}
+
+	if req.Sort == "" {
+		req.Sort = "time"
+	}
+
+	switch {
+
+	case req.Locator.SiteID != "" && req.Locator.URL != "": // find comments for site and url
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && (req.Since.IsZero() || c.Timestamp.After(req.Since))
+		})
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.UserID == "": // find last comments for site
+		if req.Limit > lastLimit || req.Limit == 0 {
+			req.Limit = lastLimit
+		}
+		if req.Since.IsZero() {
+			req.Since = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+		}
+
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return !c.Deleted && c.Timestamp.After(req.Since)
+		})
+		comments = engine.SortComments(comments, "-time")
+		if len(comments) > req.Limit {
+			comments = comments[:req.Limit]
+		}
+		return comments, nil
+
+	case req.Locator.SiteID != "" && req.UserID != "": // find comments for user
+		comments = m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID
+		})
+	}
+
+	comments = engine.SortComments(comments, req.Sort)
+	if req.Skip > 0 && req.Skip > len(comments) {
+		return []store.Comment{}, nil
+	}
+	if req.Skip > 0 && req.Skip < len(comments) {
+		comments = comments[req.Skip:]
+	}
+
+	if req.Limit > 0 && req.Limit < len(comments) {
+		comments = comments[:req.Limit]
+	}
+
+	return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *MemData) Get(req engine.GetRequest) (comment store.Comment, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.get(req.Locator, req.CommentID)
+}
+
+// Update updates comment for locator.URL with mutable part of comment
+func (m *MemData) Update(comment store.Comment) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.updateComment(comment)
+}
+
+// Count returns number of comments for post or user
+func (m *MemData) Count(req engine.FindRequest) (count int, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	switch {
+	case req.Locator.URL != "": // comment's count for post
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator && !c.Deleted
+		})
+		return len(comments), nil
+	case req.UserID != "":
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		return len(comments), nil
+	default:
+		return 0, errors.Errorf("invalid count request %+v", req)
+	}
+}
+
+// Info get post(s) meta info
+func (m *MemData) Info(req engine.InfoRequest) (res []store.PostInfo, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	res = []store.PostInfo{}
+
+	if req.Locator.URL != "" { // post info
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.Locator == req.Locator
+		})
+		if len(comments) == 0 {
+			return nil, errors.New("not found")
+		}
+		info := store.PostInfo{
+			URL:      req.Locator.URL,
+			Count:    len(comments),
+			ReadOnly: false,
+			FirstTS:  comments[0].Timestamp.UTC(),
+			LastTS:   comments[len(comments)-1].Timestamp.UTC(),
+		}
+		// set read-only from age and manual bucket
+		info.ReadOnly = req.ReadOnlyAge > 0 && !info.FirstTS.IsZero() &&
+			info.FirstTS.AddDate(0, 0, req.ReadOnlyAge).Before(time.Now())
+		if !info.ReadOnly {
+			v := m.checkFlag(engine.FlagRequest{Flag: engine.ReadOnly, Locator: req.Locator})
+			info.ReadOnly = v
+		}
+		return []store.PostInfo{info}, nil
+	}
+
+	if req.Locator.URL == "" && req.Locator.SiteID != "" { // site info (list)
+		if req.Limit <= 0 {
+			req.Limit = 1000
+		}
+		if req.Skip < 0 {
+			req.Skip = 0
+		}
+
+		infoAll := map[store.Locator]store.PostInfo{}
+		for _, c := range m.posts[req.Locator.SiteID] {
+			var info store.PostInfo
+			var ok bool
+			if info, ok = infoAll[c.Locator]; !ok {
+				info = store.PostInfo{URL: c.Locator.URL, FirstTS: c.Timestamp.UTC()}
+			}
+			info.Count++
+			info.LastTS = c.Timestamp.UTC()
+			infoAll[c.Locator] = info
+		}
+
+		for _, v := range infoAll {
+			res = append(res, v)
+		}
+		sort.Slice(res, func(i, j int) bool {
+			return res[i].URL > res[j].URL
+		})
+
+		if req.Skip > 0 {
+			if req.Skip >= len(res) {
+				return []store.PostInfo{}, nil
+			}
+			res = res[req.Skip:]
+		}
+
+		if req.Limit > 0 && req.Limit < len(res) {
+			res = res[:req.Limit]
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("invalid info request %+v", req)
+}
+
+// Flag sets and gets flag values
+func (m *MemData) Flag(req engine.FlagRequest) (val bool, err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if req.Update == engine.FlagNonSet { // read flag value, no update requested
+		return m.checkFlag(req), nil
+	}
+	// write flag value
+	return m.setFlag(req)
+}
+
+// ListFlags get list of flagged keys, like blocked & verified user
+// works for full locator (post flags) or with userID
+func (m *MemData) ListFlags(req engine.FlagRequest) (res []interface{}, err error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	res = []interface{}{}
+
+	switch req.Flag {
+	case engine.Verified:
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID {
+				res = append(res, u.UserID)
+			}
+		}
+		return res, nil
+
+	case engine.Blocked:
+		log.Printf("%+v", m.metaUsers)
+		for _, u := range m.metaUsers {
+			if u.SiteID == req.Locator.SiteID && u.Blocked && u.BlockedUntil.After(time.Now()) {
+				res = append(res, store.BlockedUser{ID: u.UserID, Until: u.BlockedUntil})
+			}
+		}
+		return res, nil
+	}
+
+	return nil, errors.Errorf("flag %s not listable", req.Flag)
+}
+
+// UserDetail sets or gets single detail value, or gets all details fo§r requested site.
+// UserDetail returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (m *MemData) UserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	switch req.Detail {
+	case engine.UserEmail, engine.UserTelegram:
+		if req.UserID == "" {
+			return nil, errors.New("userid cannot be empty in request for single detail")
+		}
+
+		m.mu.Lock()
+		defer m.mu.Unlock()
+
+		if req.Update == "" { // read detail value, no update requested
+			return m.getUserDetail(req)
+		}
+
+		return m.setUserDetail(req)
+	case engine.AllUserDetails:
+		// list of all details returned in case request is a read request
+		// (Update is not set) and does not have UserID or Detail set
+		if req.Update == "" && req.UserID == "" { // read list of all details
+			m.mu.Lock()
+			defer m.mu.Unlock()
+			return m.listDetails(req.Locator)
+		}
+		return nil, errors.New("unsupported request with userdetail all")
+	default:
+		return nil, errors.Errorf("unsupported detail %q", req.Detail)
+	}
+}
+
+// Delete post(s), user, comment, user details, or everything
+func (m *MemData) Delete(req engine.DeleteRequest) error {
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	switch {
+	case req.UserDetail != "": // delete user detail
+		return m.deleteUserDetail(req.Locator, req.UserID, req.UserDetail)
+	case req.Locator.URL != "" && req.CommentID != "" && req.UserDetail == "": // delete comment
+		return m.deleteComment(req.Locator, req.CommentID, req.DeleteMode)
+
+	case req.Locator.SiteID != "" && req.UserID != "" && req.CommentID == "" && req.UserDetail == "": // delete user
+		comments := m.match(m.posts[req.Locator.SiteID], func(c store.Comment) bool {
+			return c.User.ID == req.UserID && !c.Deleted
+		})
+		for _, c := range comments {
+			if e := m.deleteComment(c.Locator, c.ID, req.DeleteMode); e != nil {
+				return e
+			}
+		}
+		return m.deleteUserDetail(req.Locator, req.UserID, engine.AllUserDetails)
+
+	case req.Locator.SiteID != "" && req.Locator.URL == "" && req.CommentID == "" && req.UserID == "" && req.UserDetail == "": // delete site
+		if _, ok := m.posts[req.Locator.SiteID]; !ok {
+			return errors.New("not found")
+		}
+		m.posts[req.Locator.SiteID] = []store.Comment{}
+		return nil
+	}
+
+	return errors.Errorf("invalid delete request %+v", req)
+}
+
+func (m *MemData) deleteComment(loc store.Locator, id string, mode store.DeleteMode) error {
+
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == id
+	})
+	if len(comments) == 0 {
+		return errors.New("not found")
+	}
+
+	comments[0].SetDeleted(mode)
+	return m.updateComment(comments[0])
+}
+
+// Close store
+func (m *MemData) Close() error {
+	return nil
+}
+
+func (m *MemData) checkFlag(req engine.FlagRequest) (val bool) {
+	switch req.Flag {
+	case engine.Blocked:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Blocked && meta.BlockedUntil.After(time.Now())
+		}
+	case engine.Verified:
+		if meta, ok := m.metaUsers[req.UserID]; ok {
+			if meta.SiteID != req.Locator.SiteID {
+				return false
+			}
+			return meta.Verified
+		}
+	case engine.ReadOnly:
+		if meta, ok := m.metaPosts[req.Locator]; ok {
+			return meta.ReadOnly
+		}
+	}
+	return false
+}
+
+func (m *MemData) setFlag(req engine.FlagRequest) (res bool, err error) {
+
+	status := false
+	if req.Update == engine.FlagTrue {
+		status = true
+	}
+
+	switch req.Flag {
+
+	case engine.Blocked:
+		until := time.Time{}
+		if status {
+			until = time.Now().AddDate(100, 0, 0) // permanent is 100years
+			if req.TTL > 0 {
+				until = time.Now().Add(req.TTL)
+			}
+		}
+		meta := metaUser{
+			UserID:       req.UserID,
+			SiteID:       req.Locator.SiteID,
+			Blocked:      status,
+			BlockedUntil: until,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.Verified:
+		meta := metaUser{
+			UserID:   req.UserID,
+			SiteID:   req.Locator.SiteID,
+			Verified: status,
+		}
+		m.metaUsers[req.UserID] = meta
+
+	case engine.ReadOnly:
+		info, ok := m.metaPosts[req.Locator]
+		if !ok {
+			info.SiteID = req.Locator.SiteID
+			info.PostURL = req.Locator.URL
+		}
+		info.ReadOnly = status
+		m.metaPosts[req.Locator] = info
+	}
+	return status, errors.Wrapf(err, "failed to set flag %+v", req)
+}
+
+// getUserDetail returns UserDetailEntry with requested userDetail (omitting other details)
+// as an only element of the slice.
+func (m *MemData) getUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		switch req.Detail {
+		case engine.UserEmail:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Email: meta.Details.Email}}, nil
+		case engine.UserTelegram:
+			return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: meta.Details.Telegram}}, nil
+		}
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// setUserDetail sets requested userDetail, returning complete updated UserDetailEntry as an onlyIps
+// element of the slice in case of success
+func (m *MemData) setUserDetail(req engine.UserDetailRequest) ([]engine.UserDetailEntry, error) {
+	var entry metaUser
+	if meta, ok := m.metaUsers[req.UserID]; ok {
+		if meta.SiteID != req.Locator.SiteID {
+			return []engine.UserDetailEntry{}, nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) {
+		entry = metaUser{
+			UserID:  req.UserID,
+			SiteID:  req.Locator.SiteID,
+			Details: engine.UserDetailEntry{UserID: req.UserID},
+		}
+	}
+
+	switch req.Detail {
+	case engine.UserEmail:
+		entry.Details.Email = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Email: req.Update}}, nil
+	case engine.UserTelegram:
+		entry.Details.Telegram = req.Update
+		m.metaUsers[req.UserID] = entry
+		return []engine.UserDetailEntry{{UserID: req.UserID, Telegram: req.Update}}, nil
+	}
+
+	return []engine.UserDetailEntry{}, nil
+}
+
+// listDetails lists all available users details for given siteID
+func (m *MemData) listDetails(loc store.Locator) ([]engine.UserDetailEntry, error) {
+	var res []engine.UserDetailEntry
+	for _, u := range m.metaUsers {
+		if u.SiteID == loc.SiteID {
+			res = append(res, u.Details)
+		}
+	}
+	return res, nil
+}
+
+// deleteUserDetail deletes requested UserDetail or whole UserDetailEntry,
+// deletion of the absent entry doesn't produce error.
+// Trying to delete user with wrong siteID doesn't to anything and doesn't produce error.
+func (m *MemData) deleteUserDetail(locator store.Locator, userID string, userDetail engine.UserDetail) error {
+	var entry metaUser
+	if meta, ok := m.metaUsers[userID]; ok {
+		if meta.SiteID != locator.SiteID {
+			return nil
+		}
+		entry = meta
+	}
+
+	if entry == (metaUser{}) || entry.Details == (engine.UserDetailEntry{}) {
+		// absent entry means that we should not do anything
+		return nil
+	}
+
+	switch userDetail {
+	case engine.UserEmail:
+		entry.Details.Email = ""
+	case engine.UserTelegram:
+		entry.Details.Telegram = ""
+	case engine.AllUserDetails:
+		entry.Details = engine.UserDetailEntry{UserID: userID}
+	}
+
+	if entry.Details == (engine.UserDetailEntry{UserID: userID}) {
+		// no user details are stored, empty details entry altogether
+		entry.Details = engine.UserDetailEntry{}
+	}
+
+	m.metaUsers[userID] = entry
+	return nil
+}
+
+func (m *MemData) get(loc store.Locator, commentID string) (store.Comment, error) {
+	comments := m.match(m.posts[loc.SiteID], func(c store.Comment) bool {
+		return c.Locator == loc && c.ID == commentID
+	})
+	if len(comments) == 0 {
+		return store.Comment{}, errors.New("not found")
+	}
+	return comments[0], nil
+}
+
+func (m *MemData) updateComment(comment store.Comment) error {
+	comments := m.posts[comment.Locator.SiteID]
+	for i, c := range comments {
+		if c.ID != comment.ID || c.Locator != comment.Locator {
+			continue
+		}
+		c.Text = comment.Text
+		c.Orig = comment.Orig
+		c.Score = comment.Score
+		c.Votes = comment.Votes
+		c.Pin = comment.Pin
+		c.Deleted = comment.Deleted
+		c.User = comment.User
+		comments[i] = c
+		m.posts[comment.Locator.SiteID] = comments
+		return nil
+	}
+	return errors.New("not found")
+}
+
+func (m *MemData) match(comments []store.Comment, fn func(c store.Comment) bool) (res []store.Comment) {
+	res = []store.Comment{}
+	for _, c := range comments {
+		if fn(c) {
+			res = append(res, c)
+		}
+	}
+	return res
+}

backend/_example/memory_store/accessor/data_test.go

@@ -0,0 +1,819 @@
+/*
+ * Copyright 2019 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestMemData_CreateAndFind(t *testing.T) {
+	m := prepMem(t) // adds two comments
+
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text)
+	assert.Equal(t, "user1", res[0].User.ID)
+
+	_, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "dup key")
+
+	id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+	require.NoError(t, err)
+	assert.Equal(t, "id-3", id)
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, Sort: "time"}
+	res, err = m.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+}
+
+func TestMemData_CreateFailedReadOnly(t *testing.T) {
+	b := prepMem(t)
+	comment := store.Comment{
+		ID:        "id-ro",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/ro", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+
+	flagReq := engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagTrue}
+	v, err := b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, v)
+
+	_, err = b.Create(comment)
+	assert.Error(t, err)
+	assert.Equal(t, "post https://radio-t.com/ro is read-only", err.Error())
+
+	flagReq = engine.FlagRequest{Locator: comment.Locator, Flag: engine.ReadOnly, Update: engine.FlagFalse}
+	v, err = b.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, v)
+
+	_, err = b.Create(comment)
+	assert.NoError(t, err)
+}
+
+func TestMemData_Get(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment, err := b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "some text2", comment.Text)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567"))
+	assert.EqualError(t, err, `not found`)
+
+	_, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, res[1].ID))
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_Update(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "2 records initially")
+
+	comment := res[0]
+	comment.Text = "abc 123"
+	comment.Score = 100
+	err = b.Update(comment)
+	assert.NoError(t, err)
+
+	comment, err = b.Get(getReq(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID))
+	assert.NoError(t, err)
+	assert.Equal(t, "abc 123", comment.Text)
+	assert.Equal(t, res[0].ID, comment.ID)
+	assert.Equal(t, 100, comment.Score)
+
+	comment.Locator.SiteID = "bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+
+	comment.Locator.SiteID = "https://radio-t.com"
+	comment.Locator.URL = "https://radio-t.com-bad"
+	err = b.Update(comment)
+	assert.EqualError(t, err, `not found`)
+}
+
+func TestMemData_FindLast(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Limit = 1
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Locator.SiteID = "bad"
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindLastSince(t *testing.T) {
+	b := prepMem(t)
+	ts := time.Date(2017, 12, 20, 15, 18, 21, 0, time.Local)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", Since: ts}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res))
+	assert.Equal(t, "some text2", res[0].Text)
+
+	req.Since = time.Date(2017, 12, 20, 16, 18, 22, 0, time.Local)
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_FindForUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 5}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 1, len(res), "allow 1 comment")
+	assert.Equal(t, `some text, <a href="http://radio-t.com">link</a>`, res[0].Text, "second comment")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, Sort: "-time", UserID: "user1", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "userZ", Limit: 1, Skip: 1}
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res), "no comments")
+}
+
+func TestMemData_FindForUserPagination(t *testing.T) {
+	b := NewMemData()
+
+	c := store.Comment{
+		Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:    store.User{ID: "user1", Name: "user name"},
+	}
+
+	// write 200 comments
+	for i := 0; i < 200; i++ {
+		c.ID = fmt.Sprintf("idd-%d", i)
+		c.Text = fmt.Sprintf("text #%d", i)
+		c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+		_, err := b.Create(c)
+		require.NoError(t, err)
+	}
+
+	// get all comments
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "-time", UserID: "user1"}
+	res, err := b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 200, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+
+	// seek 0, 5 comments
+	req.Limit = 5
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-199", res[0].ID)
+	assert.Equal(t, "idd-195", res[4].ID)
+
+	// seek 10, 3 comments
+	req.Skip, req.Limit = 10, 3
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(res))
+	assert.Equal(t, "idd-189", res[0].ID)
+	assert.Equal(t, "idd-187", res[2].ID)
+
+	// seek 195, ask 10 comments
+	req.Skip, req.Limit = 195, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	require.Equal(t, 5, len(res))
+	assert.Equal(t, "idd-4", res[0].ID)
+	assert.Equal(t, "idd-0", res[4].ID)
+
+	// seek 255, ask 10 comments
+	req.Skip, req.Limit = 255, 10
+	res, err = b.Find(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(res))
+}
+
+func TestMemData_CountPost(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "bad"}}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	c, err = b.Count(engine.FindRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_CountUser(t *testing.T) {
+	b := prepMem(t)
+	req := engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	c, err := b.Count(req)
+	assert.NoError(t, err)
+	require.Equal(t, 2, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+
+	req = engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "userZ"}
+	c, err = b.Count(req)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c)
+}
+
+func TestMemData_InfoPost(t *testing.T) {
+	b := prepMem(t)
+	ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	req := engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err := b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 10}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1,
+		FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, r)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, ReadOnlyAge: 0}
+	_, err = b.Info(req)
+	require.Error(t, err)
+
+	_, err = b.Info(engine.InfoRequest{})
+	require.Error(t, err)
+
+	fr := engine.FlagRequest{Flag: engine.ReadOnly,
+		Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, Update: engine.FlagTrue}
+	_, err = b.Flag(fr)
+	require.NoError(t, err)
+	req = engine.InfoRequest{Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, ReadOnlyAge: 0}
+	r, err = b.Info(req)
+	require.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24),
+		ReadOnly: true}}, r)
+}
+
+func TestMemData_InfoList(t *testing.T) {
+	b := prepMem(t)
+	// add one more for https://radio-t.com/2
+	comment := store.Comment{
+		ID:        "12345",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Create(comment)
+	assert.NoError(t, err)
+
+	ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+	req := engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	res, err := b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: -1, Skip: -1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+		{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+	req = engine.InfoRequest{Locator: store.Locator{SiteID: "bad"}, Limit: 1, Skip: 1}
+	res, err = b.Info(req)
+	assert.NoError(t, err)
+	assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMemData_FlagBlockedUser(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing blocked yet")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagTrue}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.True(t, val, "user1 still blocked")
+
+	req = engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	val, err = b.Flag(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1"})
+	assert.NoError(t, err)
+	assert.False(t, val, "user1 unblocked")
+}
+
+func TestMemData_FlagReadOnlyPost(t *testing.T) {
+
+	b := prepMem(t)
+	req := engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err := b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "nothing ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagTrue}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.True(t, val, "url-1 ro")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-2"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-2 still writable")
+
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly,
+		Update: engine.FlagFalse}
+	_, err = b.Flag(req)
+	assert.NoError(t, err)
+	req = engine.FlagRequest{Locator: store.Locator{SiteID: "radio-t", URL: "url-1"}, Flag: engine.ReadOnly}
+	val, err = b.Flag(req)
+	assert.NoError(t, err)
+	assert.False(t, val, "url-1 writable")
+}
+
+func TestMemData_FlagVerified(t *testing.T) {
+
+	b := prepMem(t)
+	isVerified := func(site, user string) bool {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user}
+		v, err := b.Flag(req)
+		require.NoError(t, err)
+		return v
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	assert.False(t, isVerified("radio-t", "u1"), "nothing verified")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.True(t, isVerified("radio-t", "u1"), "u1 verified")
+
+	assert.False(t, isVerified("radio-t", "u2"), "u2 still not verified")
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagFalse))
+	assert.False(t, isVerified("radio-t", "u1"), "u1 not verified anymore")
+
+	assert.NoError(t, setVerified("bad", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u1xyz", engine.FlagFalse))
+
+	assert.False(t, isVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u3", engine.FlagFalse))
+}
+
+func TestMemData_FlagListVerified(t *testing.T) {
+
+	b := prepMem(t)
+	toIDs := func(inp []interface{}) (res []string) {
+		res = make([]string, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(string)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		sort.Strings(res)
+		return res
+	}
+
+	setVerified := func(site, user string, status engine.FlagStatus) error {
+		req := engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: site}, UserID: user, Update: status}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	ids, err := b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, toIDs(ids), "verified list empty")
+
+	assert.NoError(t, setVerified("radio-t", "u1", engine.FlagTrue))
+	assert.NoError(t, setVerified("radio-t", "u2", engine.FlagTrue))
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	assert.EqualValues(t, []string{"u1", "u2"}, toIDs(ids), "verified 2 ids")
+
+	ids, err = b.ListFlags(engine.FlagRequest{Flag: engine.Verified, Locator: store.Locator{SiteID: "radio-t-bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(ids))
+
+	ids, err = b.ListFlags(engine.FlagRequest{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(ids))
+}
+
+func TestMemData_FlagListBlocked(t *testing.T) {
+
+	b := prepMem(t)
+	setBlocked := func(site, user string, status engine.FlagStatus, ttl time.Duration) error {
+		req := engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: site}, UserID: user, Update: status,
+			TTL: ttl}
+		_, err := b.Flag(req)
+		return err
+	}
+
+	toBlocked := func(inp []interface{}) (res []store.BlockedUser) {
+		res = make([]store.BlockedUser, len(inp))
+		for i, v := range inp {
+			vv, ok := v.(store.BlockedUser)
+			require.True(t, ok)
+			res[i] = vv
+		}
+		return res
+	}
+	assert.NoError(t, setBlocked("radio-t", "user1", engine.FlagTrue, 0))
+	assert.NoError(t, setBlocked("radio-t", "user2", engine.FlagTrue, 50*time.Millisecond))
+	assert.NoError(t, setBlocked("radio-t", "user3", engine.FlagFalse, 0))
+
+	vv, err := b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+
+	blockedList := toBlocked(vv)
+	var blockedIds = make([]string, len(blockedList))
+	for i, x := range blockedList {
+		blockedIds[i] = x.ID
+	}
+	require.Equal(t, 2, len(blockedList), b.metaUsers)
+	assert.ElementsMatch(t, []string{"user1", "user2"}, blockedIds)
+	t.Logf("%+v", blockedList)
+
+	// check block expiration
+	time.Sleep(50 * time.Millisecond)
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "radio-t"}})
+	assert.NoError(t, err)
+	blockedList = toBlocked(vv)
+	require.Equal(t, 1, len(blockedList))
+	assert.Equal(t, "user1", blockedList[0].ID)
+
+	vv, err = b.ListFlags(engine.FlagRequest{Flag: engine.Blocked, Locator: store.Locator{SiteID: "bad"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(vv))
+}
+
+func TestMemData_DeleteComment(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res), "initially 2 comments")
+
+	count, err := b.Count(reqReq)
+	require.NoError(t, err)
+	require.Equal(t, 2, count, "count=2 initially")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.SoftDelete}
+
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+	assert.Equal(t, "some text2", res[1].Text)
+	assert.False(t, res[1].Deleted)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+	count, err = b.Count(reqReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+
+	delReq.CommentID = "123456"
+	err = b.Delete(delReq)
+	assert.Error(t, err)
+
+	delReq.Locator.SiteID = "bad"
+	delReq.CommentID = res[0].ID
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	delReq.Locator = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+	err = b.Delete(delReq)
+	assert.EqualError(t, err, `not found`)
+
+	err = b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}})
+	assert.Error(t, err)
+}
+
+func TestMemData_Close(t *testing.T) {
+	b := prepMem(t)
+	assert.NoError(t, b.Close())
+}
+
+func TestMemData_DeleteHard(t *testing.T) {
+
+	b := prepMem(t)
+	reqReq := engine.FindRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, Sort: "time"}
+	res, err := b.Find(reqReq)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(res), "initially 2 comments")
+
+	delReq := engine.DeleteRequest{Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		CommentID: res[0].ID, DeleteMode: store.HardDelete}
+	err = b.Delete(delReq)
+	assert.NoError(t, err)
+
+	res, err = b.Find(reqReq)
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(res))
+	assert.Equal(t, "", res[0].Text)
+	assert.True(t, res[0].Deleted, "marked deleted")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMemData_DeleteAll(t *testing.T) {
+	b := prepMem(t)
+	delReq := engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}}
+	err := b.Delete(delReq)
+	assert.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Limit: 10})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemData_UserDetailAll(t *testing.T) {
+	b := prepMem(t)
+
+	val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails})
+	require.NoError(t, err)
+	require.Nil(t, val)
+}
+
+func TestMemData_UserDetailErrors(t *testing.T) {
+	b := prepMem(t)
+
+	val, err := b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.UserEmail, Update: "value1"})
+	require.EqualError(t, err, "userid cannot be empty in request for single detail")
+	require.Nil(t, val)
+
+	val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails, Update: "value1"})
+	require.EqualError(t, err, "unsupported request with userdetail all")
+	require.Nil(t, val)
+
+	val, err = b.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: "bad"})
+	require.EqualError(t, err, "unsupported detail \"bad\"")
+	require.Nil(t, val)
+}
+
+func TestMemData_DeleteUserDetail(t *testing.T) {
+	var (
+		createEmailUser    = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail, Update: "value1"}
+		readEmailUser      = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserEmail}
+		createTelegramUser = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram, Update: "value1"}
+		readTelegramUser   = engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", Detail: engine.UserTelegram}
+		emailSet           = []engine.UserDetailEntry{{UserID: "user1", Email: "value1"}}
+		emailUnset         = []engine.UserDetailEntry{{UserID: "user1", Email: ""}}
+		telegramSet        = []engine.UserDetailEntry{{UserID: "user1", Telegram: "value1"}}
+		telegramUnset      = []engine.UserDetailEntry{{UserID: "user1", Telegram: ""}}
+	)
+
+	b := prepMem(t)
+
+	var testData = []struct {
+		delReq    engine.DeleteRequest
+		detailReq engine.UserDetailRequest
+		expected  []engine.UserDetailEntry
+	}{
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: createEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserEmail},
+			detailReq: readEmailUser, expected: emailUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: createTelegramUser, expected: telegramSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: readTelegramUser, expected: telegramSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.UserTelegram},
+			detailReq: readTelegramUser, expected: telegramUnset},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: createEmailUser, expected: emailSet},
+		{delReq: engine.DeleteRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "user1", UserDetail: engine.AllUserDetails},
+			detailReq: readEmailUser, expected: emailUnset},
+	}
+
+	for i, x := range testData {
+		err := b.Delete(x.delReq)
+		require.NoError(t, err, "delete request #%d error", i)
+
+		val, err := b.UserDetail(x.detailReq)
+		require.NoError(t, err, "user request #%d error", i)
+		require.Equal(t, x.expected, val, "user request #%d result", i)
+	}
+}
+
+func TestMemAdmin_DeleteUserHard(t *testing.T) {
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.HardDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	_, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMemAdmin_DeleteUserSoft(t *testing.T) {
+
+	b := prepMem(t)
+	err := b.Delete(engine.DeleteRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1",
+		DeleteMode: store.SoftDelete})
+	require.NoError(t, err)
+
+	comments, err := b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"},
+		Sort: "time"})
+	assert.NoError(t, err)
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[0].User)
+	assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, comments[1].User)
+
+	c, err := b.Count(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com"}})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, c, "0 count")
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, UserID: "user1", Limit: 5})
+	assert.NoError(t, err, "no comments for user user1 in store")
+	require.Equal(t, 2, len(comments), "2 comments with deleted info")
+	assert.True(t, comments[0].Deleted)
+	assert.True(t, comments[1].Deleted)
+	assert.Equal(t, "", comments[0].Text)
+	assert.Equal(t, "", comments[1].Text)
+
+	comments, err = b.Find(engine.FindRequest{Locator: store.Locator{SiteID: "radio-t"}, Sort: "time"})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func prepMem(t *testing.T) *MemData {
+
+	m := NewMemData()
+
+	comment := store.Comment{
+		ID:        "id-1",
+		Text:      `some text, <a href="http://radio-t.com">link</a>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := m.Create(comment)
+	require.NoError(t, err)
+
+	comment = store.Comment{
+		ID:        "id-2",
+		Text:      "some text2",
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err = m.Create(comment)
+	require.NoError(t, err)
+	return m
+}
+
+func getReq(locator store.Locator, commentID string) engine.GetRequest {
+	return engine.GetRequest{
+		Locator:   locator,
+		CommentID: commentID,
+	}
+}

backend/_example/memory_store/accessor/image.go

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"github.com/pkg/errors"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// MemImage implements image.Store with memory backend
+type MemImage struct {
+	imagesStaging map[string][]byte
+	images        map[string][]byte
+	insertTime    map[string]time.Time
+	mu            sync.RWMutex
+}
+
+// NewMemImageStore makes admin Store in memory.
+func NewMemImageStore() *MemImage {
+	log.Print("[DEBUG] make memory image store")
+	return &MemImage{
+		imagesStaging: map[string][]byte{},
+		images:        map[string][]byte{},
+		insertTime:    map[string]time.Time{},
+	}
+}
+
+// Save stores image with passed id to staging
+func (m *MemImage) Save(id string, img []byte) error {
+	m.mu.Lock()
+	m.imagesStaging[id] = img
+	m.insertTime[id] = time.Now()
+	m.mu.Unlock()
+
+	return nil
+}
+
+// ResetCleanupTimer resets cleanup timer for the image
+func (m *MemImage) ResetCleanupTimer(id string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if _, ok := m.insertTime[id]; ok {
+		m.insertTime[id] = time.Now()
+		return nil
+	}
+	return errors.Errorf("image %s not found", id)
+}
+
+// Load image by ID
+func (m *MemImage) Load(id string) ([]byte, error) {
+	m.mu.RLock()
+	img, ok := m.images[id]
+	if !ok {
+		img, ok = m.imagesStaging[id]
+	}
+	m.mu.RUnlock()
+	if !ok {
+		return nil, errors.Errorf("image %s not found", id)
+	}
+	return img, nil
+}
+
+// Commit moves image from staging to permanent
+func (m *MemImage) Commit(id string) error {
+	m.mu.RLock()
+	img, ok := m.imagesStaging[id]
+	m.mu.RUnlock()
+	if !ok {
+		return errors.Errorf("failed to commit %s, not found in staging", id)
+	}
+
+	m.mu.Lock()
+	m.images[id] = img
+	m.mu.Unlock()
+
+	return nil
+}
+
+// Cleanup runs removal loop for old images on staging
+func (m *MemImage) Cleanup(_ context.Context, ttl time.Duration) error {
+	var idsToRemove []string
+
+	m.mu.RLock()
+	for id, t := range m.insertTime {
+		age := time.Since(t)
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", id, age)
+			idsToRemove = append(idsToRemove, id)
+		}
+	}
+	m.mu.RUnlock()
+
+	m.mu.Lock()
+	for _, id := range idsToRemove {
+		delete(m.insertTime, id)
+		delete(m.imagesStaging, id)
+	}
+	m.mu.Unlock()
+	return nil
+}
+
+// Info returns meta information about storage
+func (m *MemImage) Info() (image.StoreInfo, error) {
+	var ts time.Time
+	m.mu.RLock()
+	for _, t := range m.insertTime {
+		if ts.IsZero() || t.Before(ts) {
+			ts = t
+		}
+	}
+	m.mu.RUnlock()
+
+	return image.StoreInfo{FirstStagingImageTS: ts}, nil
+}

backend/_example/memory_store/accessor/image_test.go

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package accessor
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+
+func TestMemImage_LoadAfterSave(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	img, err := svc.Load("test_id")
+	assert.EqualError(t, err, "image test_id not found")
+	assert.Empty(t, img)
+
+	id := "test_img"
+	err = svc.Save(id, gopher)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+
+	svc.ResetCleanupTimer(id)
+
+	err = svc.Commit(id)
+	assert.NoError(t, err)
+
+	err = svc.Cleanup(context.TODO(), 0)
+	assert.NoError(t, err)
+
+	img, err = svc.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, gopher, img)
+}
+
+func TestMemImage_CommitFail(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestMemImage_Cleanup(t *testing.T) {
+	svc := NewMemImageStore()
+	err := svc.Cleanup(context.TODO(), time.Minute)
+	assert.NoError(t, err)
+}
+
+func TestMemImage_Info(t *testing.T) {
+	svc := NewMemImageStore()
+	gopher, err := io.ReadAll(gopherPNG())
+	assert.NoError(t, err)
+
+	// get info on empty storage, should be zero
+	info, err := svc.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save image
+	err = svc.Save("test_img", gopher)
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = svc.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}

backend/_example/memory_store/compose-dev-memstore.yml

@@ -0,0 +1,64 @@
+# compose file demonstrating custom storage use. The memory_store (see backend/_example/memory_store) starts
+# in a separate container and remark42 communicates to mem_store.r42 via STORE_RPC_API url
+
+version: "2"
+
+services:
+  remark42:
+    build:
+      context: ../../..
+      dockerfile: Dockerfile
+      args:
+        - SKIP_BACKEND_TEST=true
+        - SKIP_FRONTEND_TEST=true
+    image: umputun/remark42:dev
+    container_name: "remark42-dev"
+    hostname: "remark42-dev"
+    restart: always
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    ports:
+      - "8080:8080" # primary rest server
+      - "8084:8084" # local oauth2 server
+
+    environment:
+      - REMARK_URL=http://127.0.0.1:8080
+      - SECRET=123456
+      - BACKUP_PATH=/srv/var/backup
+      - DEBUG=true
+      - EMOJI=true
+      - AUTH_ANON=true
+      - AUTH_DEV=true
+      - ADMIN_TYPE=rpc
+      - ADMIN_RPC_API=http://mem_store.r42:8080/cmd
+      - STORE_TYPE=rpc
+      - STORE_RPC_API=http://mem_store.r42:8080/cmd
+      - IMAGE_TYPE=rpc
+      - IMAGE_RPC_API=http://mem_store.r42:8080/cmd
+
+    volumes:
+      - ../../../var:/srv/var
+
+  mem_store.r42:
+    image: umputun/mem_store.r42
+    build:
+      context: ../../..
+      dockerfile: backend/_example/memory_store/Dockerfile
+    container_name: "mem_store.r42"
+    hostname: "mem_store.r42"
+
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+    environment:
+      - API=/cmd
+      - SECRET=123456
+      - DEBUG=true

backend/_example/memory_store/go.mod

@@ -0,0 +1,14 @@
+module github.com/umputun/remark42/memory_store
+
+go 1.16
+
+require (
+	github.com/go-pkgz/jrpc v0.2.0
+	github.com/go-pkgz/lgr v0.10.4
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.7.0
+	github.com/umputun/go-flags v1.5.1
+	github.com/umputun/remark42/backend v1.8.1
+)
+
+replace github.com/umputun/remark42/backend => ../../

backend/_example/memory_store/go.sum

@@ -0,0 +1,786 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
+cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
+cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
+cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
+cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
+cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
+cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Depado/bfchroma v1.3.0 h1:zz14vpvySU6S0CL6yGPr1vkFevQecIt8dJdCsMS2JpM=
+github.com/Depado/bfchroma v1.3.0/go.mod h1:c0bFk0tFmT+clD3TIGurjWCfD/QV8/EebfM3JGr+98M=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.7.3/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM=
+github.com/alecthomas/chroma v0.9.4 h1:YL7sOAE3p8HS96T9km7RgvmsZIctqbK1qJ0b7hzed44=
+github.com/alecthomas/chroma v0.9.4/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.4/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.11.4/go.mod h1:VL3UDEfAH59bSa7MuHMuFToxkqyHh69s/WUbYlOAuyg=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/oauth1 v0.7.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth/v6 v6.0.1/go.mod h1:j2pKs+JQ5PvU/K4jFnrnwntrmfUbYLJE5oSdxR37FD0=
+github.com/didip/tollbooth/v6 v6.1.1 h1:Nt7PvWLa9Y94OrykXsFNBinVRQIu8xdy4avpl99Dc1M=
+github.com/didip/tollbooth/v6 v6.1.1/go.mod h1:xjcse6CTHCLuOkzsWrEgdy9WPJFv+p/x6v+MyfP+O9s=
+github.com/didip/tollbooth_chi v0.0.0-20200524181329-8b84cd7183d9/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/didip/tollbooth_chi v0.0.0-20200828173446-a7173453ea21 h1:x7YpwKSBIBcKe9I3aTNOqgSyJ6QKDdtOxnEkxBTsi9w=
+github.com/didip/tollbooth_chi v0.0.0-20200828173446-a7173453ea21/go.mod h1:0ZVa6kSzS011nfTC1rELyxK4tjVf6vqBnOv7oY2KlsA=
+github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
+github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/chi/v5 v5.0.5/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
+github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-pkgz/auth v1.18.1-0.20211207184750-78e5d290333d/go.mod h1:N1VM53c22Uw0UQSYgdTOBK1D2vtPjSc/RWGf/Fhc1y8=
+github.com/go-pkgz/expirable-cache v0.0.3 h1:rTh6qNPp78z0bQE6HDhXBHUwqnV9i09Vm6dksJLXQDc=
+github.com/go-pkgz/expirable-cache v0.0.3/go.mod h1:+IauqN00R2FqNRLCLA+X5YljQJrwB179PfiAoMPlTlQ=
+github.com/go-pkgz/jrpc v0.2.0 h1:CLy/eZyekjraVrxZV18N2R1mYLMJ/nWrgdfyIOGPY/E=
+github.com/go-pkgz/jrpc v0.2.0/go.mod h1:wd8vtQ4CgtCnuqua6x2b1SKIgv0VSOh5Dn0uUITbiUE=
+github.com/go-pkgz/lcw v0.8.1/go.mod h1:Xw0/ZfApATgbjVPYRZO4XHdWyxAjErDWDWJ7TLlw1Vc=
+github.com/go-pkgz/lgr v0.10.4 h1:l7qyFjqEZgwRgaQQSEp6tve4A3OU80VrfzpvtEX8ngw=
+github.com/go-pkgz/lgr v0.10.4/go.mod h1:CD0s1z6EFpIUplV067gitF77tn25JItzwHNKAPqeCF0=
+github.com/go-pkgz/repeater v1.1.3/go.mod h1:hVTavuO5x3Gxnu8zW7d6sQBfAneKV8X2FjU48kGfpKw=
+github.com/go-pkgz/rest v1.5.0/go.mod h1:nQaM3RhSTUAmbBZWY4hfe4buyeC9VckvhoCktiQXJxI=
+github.com/go-pkgz/rest v1.11.0 h1:Z//qgmM0NhBYfhXYEP/aJtDVLK5XlJGxqcb4sHFNN0E=
+github.com/go-pkgz/rest v1.11.0/go.mod h1:wZ/dGipZUaF9to0vIQl7PwDHgWQDB0jsrFg1xnAKLDw=
+github.com/go-pkgz/syncs v1.1.1/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kyokomi/emoji/v2 v2.2.8/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
+github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/slack-go/slack v0.9.5/go.mod h1:wWL//kk0ho+FcQXcBTmEafUI5dz4qz5f4mMk8oIkioQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/btree v0.0.0-20170113224114-9876f1454cf0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/buntdb v1.1.0/go.mod h1:Y39xhcDW10WlyYXeLgGftXVbjtM0QP+/kpz8xl9cbzE=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
+github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/umputun/go-flags v1.5.1 h1:vRauoXV3Ultt1HrxivSxowbintgZLJE+EcBy5ta3/mY=
+github.com/umputun/go-flags v1.5.1/go.mod h1:nTbvsO/hKqe7Utri/NoyN18GR3+EWf+9RrmsdwdhrEc=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.mongodb.org/mongo-driver v1.7.4/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211105192438-b53810dc28af h1:SMeNJG/vclJ5wyBBd4xupMsSJIHTd1coW9g7q6KOjmY=
+golang.org/x/net v0.0.0-20211105192438-b53810dc28af/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c h1:+8miTPjMCTXwih7BQmvWwd0PjdBZq2MKp/qQaahSzEM=
+golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
+google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
+google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
+google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
+google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
+google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
+google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
+google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
+google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/oauth2.v3 v3.12.0/go.mod h1:XEYgKqWX095YiPT+Aw5y3tCn+7/FMnlTFKrupgSiJ3I=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

backend/_example/memory_store/main.go

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-pkgz/jrpc"
+	log "github.com/go-pkgz/lgr"
+	"github.com/umputun/go-flags"
+
+	"github.com/umputun/remark42/memory_store/accessor"
+	"github.com/umputun/remark42/memory_store/server"
+)
+
+// opts with all cli commands and flags
+var opts struct {
+	API        string `long:"api" env:"API" default:"/" description:"api root url"`
+	Port       int    `long:"port" env:"PORT" default:"8080" description:"rpc server port"`
+	AuthUser   string `long:"auth-user" env:"AUTH_USER" default:"" description:"rpc auth user name"`
+	AuthPasswd string `long:"auth-passwd" env:"AUTH_PASSWD" default:"" description:"rpc auth password"`
+
+	Secret string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
+	Dbg    bool   `long:"dbg" env:"DEBUG" description:"debug mode"`
+}
+
+var revision = "unknown"
+
+func main() {
+	fmt.Printf("remark42-memory module %s\n", revision)
+
+	if _, err := flags.Parse(&opts); err != nil {
+		os.Exit(2)
+	}
+	setupLog(opts.Dbg)
+
+	dataStore := accessor.NewMemData()
+	adminStore := accessor.NewMemAdminStore(opts.Secret)
+	imgStore := accessor.NewMemImageStore()
+
+	rpcServer := jrpc.Server{
+		API:        opts.API,
+		AuthUser:   opts.AuthUser,
+		AuthPasswd: opts.AuthPasswd,
+		Version:    revision,
+		AppName:    "remark42-memory",
+		Logger:     log.Default(),
+	}
+
+	srv := server.NewRPC(dataStore, adminStore, imgStore, &rpcServer)
+
+	admRec := accessor.AdminRec{
+		SiteID:  "remark",
+		Enabled: true,
+		IDs:     []string{"dev_user"},
+		Email:   "admin@example.com",
+	}
+	adminStore.Set("remark", admRec)
+
+	err := srv.Run(opts.Port)
+	log.Printf("[ERROR] server failed or terminated, %+v", err)
+}
+
+func setupLog(dbg bool) {
+	if dbg {
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
+		return
+	}
+	log.Setup(log.Msec, log.LevelBraces)
+}

backend/_example/memory_store/server/admin.go

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+// get admin key
+func (s *RPC) admKeyHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	key, err := s.adm.Key(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, key, err)
+}
+
+// get admins list
+func (s *RPC) admAdminsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	admins, err := s.adm.Admins(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, admins, err)
+}
+
+// get admin email
+func (s *RPC) admEmailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	email, err := s.adm.Email(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, email, err)
+}
+
+// return site enabled status
+func (s *RPC) admEnabledHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	if err := json.Unmarshal(params, &siteID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+
+	ok, err := s.adm.Enabled(siteID)
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, ok, err)
+}
+
+// onEvent returns nothing, callback to OnEvent
+func (s *RPC) admEventHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var siteID string
+	var ps []interface{}
+	if err := json.Unmarshal(params, &ps); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	siteID, ok := ps[0].(string)
+	if !ok {
+		return jrpc.Response{Error: "wrong siteID type"}
+	}
+	evType, ok := ps[1].(float64)
+	if !ok {
+		return jrpc.Response{Error: "wrong event type"}
+	}
+	err := s.adm.OnEvent(siteID, admin.EventType(evType))
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.EncodeResponse(id, nil, err)
+}

backend/_example/memory_store/server/admin_test.go

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/admin"
+)
+
+func TestRPC_admKeyHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	key, err := ra.Key("any")
+	assert.NoError(t, err)
+	assert.Equal(t, "secret", key)
+}
+
+func TestRPC_admAdminsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	admins, err := ra.Admins("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"id1", "id2"}, admins)
+}
+
+func TestRPC_admEmailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Admins("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	email, err := ra.Email("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, "admin@example.com", email)
+}
+
+func TestRPC_admEnabledHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	_, err := ra.Enabled("bad site")
+	assert.EqualError(t, err, "site bad site not found")
+
+	ok, err := ra.Enabled("test-site")
+	assert.NoError(t, err)
+	assert.Equal(t, true, ok)
+
+	ok, err = ra.Enabled("test-site-disabled")
+	assert.NoError(t, err)
+	assert.Equal(t, false, ok)
+}
+
+func TestRPC_admEventHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ra := admin.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ra.OnEvent("bad site", admin.EvCreate)
+	assert.EqualError(t, err, "site bad site not found")
+
+	err = ra.OnEvent("test-site", admin.EvCreate)
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/data.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/go-pkgz/jrpc"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func (s *RPC) createHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	commentID, err := s.eng.Create(comment)
+	return jrpc.EncodeResponse(id, commentID, err)
+}
+
+// Find comments
+func (s *RPC) findHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	resp, err := s.eng.Find(req)
+	return jrpc.EncodeResponse(id, resp, err)
+}
+
+// Get comment
+func (s *RPC) getHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.GetRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	comment, err := s.eng.Get(req)
+	return jrpc.EncodeResponse(id, comment, err)
+}
+
+// Update comment
+func (s *RPC) updateHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	comment := store.Comment{}
+	if err := json.Unmarshal(params, &comment); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Update(comment)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// counts for site and users
+func (s *RPC) countHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FindRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	count, err := s.eng.Count(req)
+	return jrpc.EncodeResponse(id, count, err)
+}
+
+// info get post meta info
+func (s *RPC) infoHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.InfoRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	info, err := s.eng.Info(req)
+	return jrpc.EncodeResponse(id, info, err)
+}
+
+// flagHndl get and sets flag value
+func (s *RPC) flagHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	status, err := s.eng.Flag(req)
+	return jrpc.EncodeResponse(id, status, err)
+}
+
+// listFlagsHndl list flags for given request
+func (s *RPC) listFlagsHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.FlagRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	flags, err := s.eng.ListFlags(req)
+	return jrpc.EncodeResponse(id, flags, err)
+}
+
+// userDetailHndl sets or gets single detail value, or gets all details for requested site.
+// userDetailHndl returns list even for single entry request is a compromise in order to have both single detail getting and setting
+// and all site's details listing under the same function (and not to extend engine interface by two separate functions).
+func (s *RPC) userDetailHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.UserDetailRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.eng.UserDetail(req)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+// deleteHndl delete post(s), user, comment, user details, or everything
+func (s *RPC) deleteHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	req := engine.DeleteRequest{}
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.eng.Delete(req)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+// close store
+func (s *RPC) closeHndl(_ uint64, _ json.RawMessage) (rr jrpc.Response) {
+	if err := s.eng.Close(); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	return jrpc.Response{}
+}

backend/_example/memory_store/server/data_test.go

@@ -0,0 +1,306 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/umputun/remark42/backend/app/store"
+	"github.com/umputun/remark42/backend/app/store/engine"
+)
+
+func TestRPC_createHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	id, err := re.Create(store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}})
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+}
+
+func TestRPC_findHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	comments, err := re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(comments))
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	comments, err = re.Find(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(comments))
+	assert.Equal(t, c, comments[0])
+}
+
+func TestRPC_getHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	_, err := re.Get(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_updateHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	err := re.Update(c)
+	assert.EqualError(t, err, "not found")
+
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	c.Text = "updates"
+	err = re.Update(c)
+	assert.NoError(t, err)
+
+	req := engine.GetRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+	comment, err := re.Get(req)
+	assert.NoError(t, err)
+	assert.Equal(t, c, comment)
+}
+
+func TestRPC_countHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	findReq := engine.FindRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	count, err := re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count)
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	count, err = re.Count(findReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+}
+
+func TestRPC_infoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	infoReq := engine.InfoRequest{Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"}}
+	info, err := re.Info(infoReq)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(info))
+	i := info[0]
+	assert.Equal(t, store.PostInfo{URL: "http://example.com/post1", Count: 1}, i)
+}
+
+func TestRPC_flagHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag: engine.Verified,
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+		UserID: "u1",
+	}
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, false, status)
+
+	flagReq.Update = engine.FlagTrue
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flagReq.Update = engine.FlagNonSet
+	status, err = re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+}
+
+func TestRPC_listFlagsHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	id, err := re.Create(c)
+	assert.NoError(t, err)
+	assert.Equal(t, "123456", id)
+
+	flagReq := engine.FlagRequest{
+		Flag:   engine.Verified,
+		UserID: "u1",
+		Locator: store.Locator{
+			SiteID: "test-site",
+		},
+	}
+	flags, err := re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{}, flags)
+
+	flagReq.Update = engine.FlagTrue
+	status, err := re.Flag(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, true, status)
+
+	flags, err = re.ListFlags(flagReq)
+	require.NoError(t, err)
+	assert.Equal(t, []interface{}{"u1"}, flags)
+}
+
+func TestRPC_userDetailHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// add to entries to DB before we start
+	result, err := re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail, Update: "test@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}, result)
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u2", Detail: engine.UserEmail, Update: "other@example.com"})
+	assert.NoError(t, err, "No error inserting entry expected")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{{UserID: "u2", Email: "other@example.com"}}, result)
+
+	// try to change existing entry with wrong SiteID
+	result, err = re.UserDetail(engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u2", Detail: engine.UserEmail, Update: "not_relevant"})
+	assert.NoError(t, err, "Updating existing entry with wrong SiteID doesn't produce error")
+	assert.ElementsMatch(t, []engine.UserDetailEntry{}, result, "Updating existing entry with wrong SiteID doesn't change anything")
+
+	// stateless tests without changing the state we set up before
+	var testData = []struct {
+		req      engine.UserDetailRequest
+		error    string
+		expected []engine.UserDetailEntry
+	}{
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "bad"}, UserID: "u1", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, UserID: "u1xyz", Detail: engine.UserEmail},
+			expected: []engine.UserDetailEntry{}},
+		{req: engine.UserDetailRequest{Detail: engine.UserEmail, Update: "new_value"},
+			error: `userid cannot be empty in request for single detail`},
+		{req: engine.UserDetailRequest{Detail: engine.UserDetail("bad")},
+			error: `unsupported detail "bad"`},
+		{req: engine.UserDetailRequest{Update: "not_relevant", Detail: engine.AllUserDetails},
+			error: `unsupported request with userdetail all`},
+		{req: engine.UserDetailRequest{Locator: store.Locator{SiteID: "test-site"}, Detail: engine.AllUserDetails},
+			expected: []engine.UserDetailEntry{{UserID: "u1", Email: "test@example.com"}, {UserID: "u2", Email: "other@example.com"}}},
+	}
+
+	for i, x := range testData {
+		result, err := re.UserDetail(x.req)
+		if x.error != "" {
+			assert.EqualError(t, err, x.error, "Error should match expected for case %d", i)
+		} else {
+			assert.NoError(t, err, "Error is not expected expected for case %d", i)
+		}
+		assert.ElementsMatch(t, x.expected, result, "Result should match expected for case %d", i)
+	}
+}
+
+func TestRPC_deleteHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	req := engine.DeleteRequest{
+		Locator:   store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		CommentID: "123456",
+	}
+
+	err := re.Delete(req)
+	assert.EqualError(t, err, "not found")
+
+	c := store.Comment{ID: "123456", Locator: store.Locator{SiteID: "test-site", URL: "http://example.com/post1"},
+		Text: "text 123", User: store.User{ID: "u1", Name: "user1"}}
+	_, err = re.Create(c)
+	assert.NoError(t, err)
+
+	err = re.Delete(req)
+	assert.NoError(t, err)
+}
+
+func TestRPC_closeHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	re := engine.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := re.Close()
+	assert.NoError(t, err)
+}

backend/_example/memory_store/server/image.go

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+)
+
+func (s *RPC) imgSaveWithIDHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var req [2]string
+	if err := json.Unmarshal(params, &req); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	img, err := base64.StdEncoding.DecodeString(req[1])
+	if err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err = s.img.Save(req[0], img)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgResetClnTimerHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.ResetCleanupTimer(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+
+}
+
+func (s *RPC) imgLoadHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	value, err := s.img.Load(fileID)
+	return jrpc.EncodeResponse(id, value, err)
+}
+
+func (s *RPC) imgCommitHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var fileID string
+	if err := json.Unmarshal(params, &fileID); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Commit(fileID)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgCleanupHndl(id uint64, params json.RawMessage) (rr jrpc.Response) {
+	var ttl time.Duration
+	if err := json.Unmarshal(params, &ttl); err != nil {
+		return jrpc.Response{Error: err.Error()}
+	}
+	err := s.img.Cleanup(context.TODO(), ttl)
+	return jrpc.EncodeResponse(id, nil, err)
+}
+
+func (s *RPC) imgInfoHndl(id uint64, _ json.RawMessage) (rr jrpc.Response) {
+	info, err := s.img.Info()
+	return jrpc.EncodeResponse(id, info, err)
+}

backend/_example/memory_store/server/image_test.go

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2020 Umputun. All rights reserved.
+ * Use of this source code is governed by a MIT-style
+ * license that can be found in the LICENSE file.
+ */
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-pkgz/jrpc"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/umputun/remark42/backend/app/store/image"
+)
+
+// gopher png for test, from https://golang.org/src/image/png/example_test.go
+const gopher = "iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2" +
+	"+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba" +
+	"73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt" +
+	"/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDs" +
+	"JkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk" +
+	"8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAA" +
+	"ILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36I" +
+	"MRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2" +
+	"MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZm" +
+	"rSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhkt" +
+	"rRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0" +
+	"dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEj" +
+	"yy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67" +
+	"LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR" +
+	"11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5fe" +
+	"A+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fU" +
+	"yLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy" +
+	"1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2S" +
+	"nssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg=="
+
+func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) }
+func gopherPNGBytes() []byte {
+	img, _ := io.ReadAll(gopherPNG())
+	return img
+}
+
+func TestRPC_imgLoadHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// commit
+	err = ri.Commit(id)
+	assert.NoError(t, err)
+
+	// load after commit
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// cleanup
+	err = ri.Cleanup(context.TODO(), time.Second)
+	assert.NoError(t, err)
+
+	// load after cleanup
+	img, err = ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+}
+
+func TestRPC_imgCommitHndlFail(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+	err := ri.Commit("test_id")
+	assert.EqualError(t, err, "failed to commit test_id, not found in staging")
+}
+
+func TestRPC_imgCleanupHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// save
+	id := "test_img"
+	err := ri.Save(id, gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// load
+	img, err := ri.Load(id)
+	assert.NoError(t, err)
+	assert.Equal(t, 1462, len(img))
+	assert.Equal(t, gopherPNGBytes(), img)
+
+	// wait for image to expire
+	time.Sleep(time.Millisecond * 50)
+	// reset the time to cleanup
+	err = ri.ResetCleanupTimer(id)
+	assert.NoError(t, err)
+
+	// cleanup, should not affect the new image
+	err = ri.Cleanup(context.TODO(), time.Millisecond*45)
+	assert.NoError(t, err)
+
+	// load after cleanup should succeed
+	_, err = ri.Load(id)
+	assert.NoError(t, err, "image is still on staging because it's cleanup timer was reset")
+
+	// cleanup with short TTL, should remove the image from staging
+	err = ri.Cleanup(context.TODO(), time.Nanosecond)
+	assert.NoError(t, err)
+
+	// load after cleanup should fail
+	_, err = ri.Load(id)
+	assert.EqualError(t, err, "image test_img not found")
+}
+
+func TestRPC_imgInfoHndl(t *testing.T) {
+	port, teardown := prepTestStore(t)
+	defer teardown()
+	api := fmt.Sprintf("http://localhost:%d/test", port)
+
+	ri := image.RPC{Client: jrpc.Client{API: api, Client: http.Client{Timeout: 1 * time.Second}}}
+
+	// get info on empty storage, should be zero
+	info, err := ri.Info()
+	assert.NoError(t, err)
+	assert.True(t, info.FirstStagingImageTS.IsZero())
+
+	// save
+	err = ri.Save("test_img", gopherPNGBytes())
+	assert.NoError(t, err)
+
+	// get info after saving, should be non-zero
+	info, err = ri.Info()
+	assert.NoError(t, err)
+	assert.False(t, info.FirstStagingImageTS.IsZero())
+}