From a1450cff575b4ce24aa9450b84e164747984e0c9 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 10 Mar 2019 19:40:14 -0500
Subject: [PATCH 01/45] add image storage

---
 backend/app/store/image/image.go      | 108 ++++++++++++++++++++++++++
 backend/app/store/image/image_test.go |  94 ++++++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 backend/app/store/image/image.go
 create mode 100644 backend/app/store/image/image_test.go

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
new file mode 100644
index 00000000..80ec9fb2
--- /dev/null
+++ b/backend/app/store/image/image.go
@@ -0,0 +1,108 @@
+// Package image handles storing, resizing and retrival of images
+package image
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"math"
+	"os"
+	"path"
+	"strconv"
+	"sync"
+
+	log "github.com/go-pkgz/lgr"
+
+	"github.com/pkg/errors"
+)
+
+// Interface defines Save and Load methods
+type Interface interface {
+	Save(name string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Load(id string) (io.ReadCloser, error)                // load image by ID. Caller has to close the reader.
+}
+
+// FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
+type FileSystem struct {
+	Location  string
+	MaxSize   int
+	Partitons int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS. Returns id as a hash of name
+// name should be passed in unique prefix, for example with userID_*
+func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
+
+	h := sha1.Sum([]byte(name))
+	id = hex.EncodeToString(h[:])
+	if ext := path.Ext(name); ext != "" {
+		id += ext
+	}
+	location := f.location(id)
+	dst := path.Join(location, id)
+
+	if err := os.MkdirAll(location, 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	fh, err := os.Create(dst)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't make image file %s", dst)
+	}
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	written, err := io.Copy(fh, lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+	if err := fh.Close(); err != nil {
+		return "", errors.Wrapf(err, "can't close image file %s", dst)
+	}
+	if written > int64(f.MaxSize) {
+		if err = os.Remove(dst); err != nil {
+			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
+		}
+		return "", errors.Errorf("file %s is too large", name)
+	}
+	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), name)
+	return id, nil
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, error) {
+	location := f.location(id)
+	imgFile := path.Join(location, id)
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, nil
+}
+
+// get location (directory) for id by adding partition to the final path in order to keep files
+// in different subdirectories and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/92. Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(id string) string {
+	if f.Partitons == 0 {
+		return f.Location
+	}
+
+	f.crc.Do(func() {
+		f.crc.Table = crc64.MakeTable(crc64.ECMA)
+		p := int(math.Round(math.Log10(float64(f.Partitons))))
+		f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+		f.crc.divider = uint64(math.Pow(10, float64(p)))
+	})
+	checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+	partition := checksum64 % f.crc.divider
+	return path.Join(f.Location, fmt.Sprintf(f.crc.mask, partition))
+}
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
new file mode 100644
index 00000000..94f6358c
--- /dev/null
+++ b/backend/app/store/image/image_test.go
@@ -0,0 +1,94 @@
+package image
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImage_Save(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   50,
+	}
+	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	assert.Equal(t, "6851dcde6024e03258a66705f29e14b506048c74.png", id)
+
+	dst := path.Join(loc, "02", id)
+	data, err := ioutil.ReadFile(dst)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestImage_SaveTooLarge(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   5,
+	}
+	_, err = svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.Error(t, err)
+	assert.EqualError(t, err, "file blah_ff1.png is too large")
+}
+
+func TestImage_Load(t *testing.T) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+	defer os.RemoveAll(loc)
+
+	// save image
+	svc := FileSystem{
+		Location:  loc,
+		Partitons: 100,
+		MaxSize:   50,
+	}
+	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+
+	r, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer r.Close()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+
+	_, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestImage_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "abcdefg", "/tmp/2"},
+		{10, "abcdefe", "/tmp/1"},
+		{10, "12345", "/tmp/9"},
+		{100, "12345", "/tmp/69"},
+		{100, "xyzz", "/tmp/58"},
+		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/02"},
+		{0, "12345", "/tmp"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitons: tt.partitions}
+			assert.Equal(t, tt.res, svc.location(tt.id))
+		})
+	}
+}

From 639f6c15f3cccb8ac2b24a7d82be1267a22f99a3 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 01:16:25 -0500
Subject: [PATCH 02/45] wire image save/load to rest and cmd

---
 .gitignore                                |   3 +-
 backend/app/cmd/server.go                 |  32 +++++++
 backend/app/rest/api/rest.go              |  18 +++-
 backend/app/rest/api/rest_private.go      |  26 ++++++
 backend/app/rest/api/rest_private_test.go |  42 +++++++++
 backend/app/rest/api/rest_public.go       |  44 ++++++++++
 backend/app/rest/api/rest_test.go         |   8 ++
 backend/app/store/image/image.go          |  14 ++-
 backend/app/store/image/image_test.go     |   6 +-
 backend/go.mod                            |   7 +-
 backend/go.sum                            |  15 ++--
 go.mod                                    |   8 ++
 go.sum                                    | 102 ++++++++++++++++++++++
 13 files changed, 304 insertions(+), 21 deletions(-)
 create mode 100644 go.mod
 create mode 100644 go.sum

diff --git a/.gitignore b/.gitignore
index 1ab85900..ec8cdf79 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,4 +17,5 @@ debug.test
 .DS_Store
 .mongo
 remark42
-/bin/
\ No newline at end of file
+/bin/
+/backend/var/
diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 6579398a..8247a900 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -32,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -43,6 +44,7 @@ type ServerCommand struct {
 	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
+	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
 	SSL    SSLGroup    `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 
 	Sites           []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
@@ -93,6 +95,19 @@ type StoreGroup struct {
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
 }
 
+// ImageGroup defines options group for store pictures
+type ImageGroup struct {
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	FS   struct {
+		Path      string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Partitons int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+	} `group:"fs" namespace:"fs" env-namespace:"FS"`
+	Bolt struct {
+		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
+	} `group:"bolt" namespace:"bolt" env-namespace:"bolt"`
+	MaxSize int `long:"max-size" env:"MAX_SIZE" default:"5000000" description:"max size of image file"`
+}
+
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
@@ -256,6 +271,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
 	commentFormatter := store.NewCommentFormatter(imgProxy)
 
+	pictStore, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make config of ssl server params")
@@ -276,6 +296,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
+		ImageService:     pictStore,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -405,6 +426,17 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
+func (s *ServerCommand) makePicturesStore() (image.Interface, error) {
+	switch s.Image.Type {
+	case "fs":
+		if err := makeDirs(s.Image.FS.Path); err != nil {
+			return nil, err
+		}
+		return &image.FileSystem{Location: s.Image.FS.Path, Partitons: s.Image.FS.Partitons, MaxSize: s.Image.MaxSize}, nil
+	}
+	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
+}
+
 func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 	log.Printf("[INFO] make admin store, type=%s", s.Admin.Type)
 
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index ccad9e37..e09a458e 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -30,6 +30,7 @@ import (
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/proxy"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -44,6 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
+	ImageService     image.Interface
 
 	WebRoot         string
 	RemarkURL       string
@@ -219,6 +221,7 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
+			ropen.Get("/picture/{id}", s.loadPictureCtrl)
 
 			ropen.Mount("/rss", s.rssRoutes())
 			ropen.Mount("/img", s.ImageProxy.Routes())
@@ -253,12 +256,25 @@ func (s *Rest) routes() chi.Router {
 			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
 			rauth.Post("/deleteme", s.deleteMeCtrl)
 		})
+
+		rapi.Group(func(rauth chi.Router) {
+			lmt := 10.0
+			if s.UpdateLimiter > 0 {
+				lmt = s.UpdateLimiter
+			}
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(authMiddleware.Auth)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Post("/picture", s.savePictureCtrl)
+		})
+
 	})
 
 	// respond to /robots.txt with the list of allowed paths
 	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
 		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img", "/avatar"}
+			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+				"/img", "/avatar", "/picture"}
 			for i := range allowed {
 				allowed[i] = "Allow: /api/v1" + allowed[i]
 			}
diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go
index add6fd9d..ec17967d 100644
--- a/backend/app/rest/api/rest_private.go
+++ b/backend/app/rest/api/rest_private.go
@@ -285,6 +285,32 @@ func (s *Rest) deleteMeCtrl(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, R.JSON{"site": siteID, "user_id": user.ID, "token": tokenStr, "link": link})
 }
 
+// POST /image - save image with form request
+func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
+	user := rest.MustGetUserInfo(r)
+
+	if err := r.ParseMultipartForm(5 * 1024 * 1024); err != nil { // 5M max memory, if bigger will make a file
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't parse multipart form", rest.ErrDecode)
+		return
+	}
+
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get image file from the request", rest.ErrInternal)
+		return
+	}
+	defer func() { _ = file.Close() }()
+
+	picName := fmt.Sprintf("%s_%d_%s", user.ID, time.Now().Nanosecond(), header.Filename)
+	id, err := s.ImageService.Save(picName, file)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
+		return
+	}
+
+	render.JSON(w, r, R.JSON{"location": id})
+}
+
 func (s *Rest) isReadOnly(locator store.Locator) bool {
 	if s.ReadOnlyAge > 0 {
 		// check RO by age
diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index 6131bfb0..46a92ad6 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -1,10 +1,13 @@
 package api
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"strings"
 	"testing"
@@ -488,3 +491,42 @@ func TestRest_DeleteMe(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 401, resp.StatusCode)
 }
+
+func TestRest_SavePictureCtrl(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	// save picture
+	r := strings.NewReader("file content 123")
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	fileWriter, err := bodyWriter.CreateFormFile("file", "picture.png")
+	require.NoError(t, err)
+	_, err = io.Copy(fileWriter, r)
+	require.NoError(t, err)
+	contentType := bodyWriter.FormDataContentType()
+	require.NoError(t, bodyWriter.Close())
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+	req.Header.Add("Content-Type", contentType)
+	req.Header.Add("X-JWT", devToken)
+	resp, err := client.Do(req)
+	assert.Nil(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err := ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+
+	m := map[string]string{}
+	err = json.Unmarshal(body, &m)
+	assert.Contains(t, m["location"], ".png")
+
+	// load picture
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, m["location"]))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	body, err = ioutil.ReadAll(resp.Body)
+	require.Nil(t, err)
+	assert.Equal(t, "file content 123", string(body))
+	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+}
diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go
index 40c8e99e..6aae3850 100644
--- a/backend/app/rest/api/rest_public.go
+++ b/backend/app/rest/api/rest_public.go
@@ -3,6 +3,7 @@ package api
 import (
 	"crypto/sha1" //nolint
 	"encoding/base64"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -330,3 +331,46 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't render posts lits for site %s", siteID)
 	}
 }
+
+// GET /picture/{id} - get picture
+func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
+
+	imgContentType := func(img string) string {
+		img = strings.ToLower(img)
+		switch {
+		case strings.HasSuffix(img, ".png"):
+			return "image/png"
+		case strings.HasSuffix(img, ".jpg") || strings.HasSuffix(img, ".jpeg"):
+			return "image/jpeg"
+		case strings.HasSuffix(img, ".gif"):
+			return "image/gif"
+		}
+		return "image/*"
+	}
+
+	id := chi.URLParam(r, "id")
+	imgRdr, size, err := s.ImageService.Load(id)
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
+		return
+	}
+	// enforce client-side caching
+	etag := `"` + id + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=604800") // 7 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	defer imgRdr.Close()
+
+	w.Header().Set("Content-Type", imgContentType(id))
+	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
+	w.WriteHeader(http.StatusOK)
+	if _, err = io.Copy(w, imgRdr); err != nil {
+		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
+	}
+}
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index d21b1c6c..c8663ac0 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/umputun/remark/backend/app/store"
 	adminstore "github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -203,6 +204,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 	os.Remove(testDb)
 	os.Remove(testHTML)
 	os.RemoveAll("/tmp/ava-remark42")
+	os.RemoveAll("/tmp/pics-remark42")
 
 	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
 	require.Nil(t, err)
@@ -232,6 +234,11 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
+		ImageService: &image.FileSystem{
+			Location:  "/tmp/pics-remark42",
+			Partitons: 100,
+			MaxSize:   10000,
+		},
 
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
@@ -258,6 +265,7 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		os.Remove(testDb)
 		os.Remove(testHTML)
 		os.RemoveAll("/tmp/ava-remark42")
+		os.RemoveAll("/tmp/pics-remark42")
 	}
 
 	return ts, srv, teardown
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 80ec9fb2..f24f6b6e 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -21,7 +21,7 @@ import (
 // Interface defines Save and Load methods
 type Interface interface {
 	Save(name string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
-	Load(id string) (io.ReadCloser, error)                // load image by ID. Caller has to close the reader.
+	Load(id string) (io.ReadCloser, int64, error)         // load image by ID. Caller has to close the reader.
 }
 
 // FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
@@ -78,14 +78,20 @@ func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 
 // Load image from FS. Uses id to get partition subdirectory.
 // returns ReadCloser and caller should call close after processing completed.
-func (f *FileSystem) Load(id string) (io.ReadCloser, error) {
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 	location := f.location(id)
 	imgFile := path.Join(location, id)
+
+	st, err := os.Stat(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image size for %s", id)
+	}
+
 	fh, err := os.Open(imgFile)
 	if err != nil {
-		return nil, errors.Wrapf(err, "can't load image %s", id)
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
 	}
-	return fh, nil
+	return fh, st.Size(), nil
 }
 
 // get location (directory) for id by adding partition to the final path in order to keep files
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 94f6358c..fd38699a 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -61,14 +61,14 @@ func TestImage_Load(t *testing.T) {
 	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
 
-	r, err := svc.Load(id)
+	r, sz, err := svc.Load(id)
 	assert.NoError(t, err)
 	defer r.Close()
 	data, err := ioutil.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))
-
-	_, err = svc.Load("abcd")
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
 	assert.NotNil(t, err)
 }
 
diff --git a/backend/go.mod b/backend/go.mod
index afe3534f..65c95d3d 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -22,18 +22,17 @@ require (
 	github.com/go-pkgz/syncs v1.1.0
 	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
 	github.com/gorilla/feeds v1.1.0
-	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
 	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.8.1
 	github.com/rakyll/statik v0.1.3
-	github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16
 	golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0
-	golang.org/x/time v0.0.0-20170927054726-6dc17368e09b // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	gopkg.in/russross/blackfriday.v2 v2.0.0
 )
diff --git a/backend/go.sum b/backend/go.sum
index 8bf05b82..f7ad9620 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -49,14 +49,12 @@ github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqC
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
 github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
 github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -77,12 +75,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
 github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
@@ -99,8 +98,8 @@ golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYb
 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
diff --git a/go.mod b/go.mod
new file mode 100644
index 00000000..a95adc32
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,8 @@
+module github.com/umputun/remark
+
+go 1.12
+
+require (
+	github.com/jessevdk/go-flags v1.4.0 // indirect
+	github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e // indirect
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 00000000..9d5c8b96
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,102 @@
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
+github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
+github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
+github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
+github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
+github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
+github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-chi/chi v3.3.2+incompatible h1:uQNcQN3NsV1j4ANsPh42P4ew4t6rnRbJb8frvpp31qQ=
+github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
+github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
+github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
+github.com/go-pkgz/auth v0.4.2 h1:WY3XzjUieUGxJSjXDU0rLrKt8RcPEaGdtLVRS7M56wU=
+github.com/go-pkgz/auth v0.4.2/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
+github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
+github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.4.0 h1:s4490VXkaepbkMZBNZgr3rgfUg0G4nOLVa/Yp0hlwyc=
+github.com/go-pkgz/lgr v0.4.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
+github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
+github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
+github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
+github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
+github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
+github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
+github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
+github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
+github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
+github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
+github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
+github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
+github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
+github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e h1:QLjdh6YuFoVtO3yrIG9ya7zPaaIa5lSUftkRew7+Joo=
+github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e/go.mod h1:PORk4Y+iHyF5RDqcp1axFaMMFZ+TfA88aYXMvkOz9pc=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
+golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
+golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
+golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
+gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=

From 3dc20bcbb5eb4b03acf3f6b3106d866322bcada0 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 01:18:36 -0500
Subject: [PATCH 03/45] image comments

---
 backend/app/store/image/image.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index f24f6b6e..6f395c36 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -14,7 +14,6 @@ import (
 	"sync"
 
 	log "github.com/go-pkgz/lgr"
-
 	"github.com/pkg/errors"
 )
 
@@ -40,6 +39,7 @@ type FileSystem struct {
 
 // Save data from reader for given file name to local FS. Returns id as a hash of name
 // name should be passed in unique prefix, for example with userID_*
+// Files partitioned across multiple subdirectories.
 func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 
 	h := sha1.Sum([]byte(name))

From 4def2affd87bd2b68b1599adbd706526baf93168 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 01:23:13 -0500
Subject: [PATCH 04/45] fix robosts test with added picture

---
 backend/app/rest/api/rest_public_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go
index 4285b384..788f34e2 100644
--- a/backend/app/rest/api/rest_public_test.go
+++ b/backend/app/rest/api/rest_public_test.go
@@ -442,5 +442,5 @@ func TestRest_Robots(t *testing.T) {
 	assert.Equal(t, 200, code)
 	assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n"+
 		"Allow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\n"+
-		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\n", string(body))
+		"Allow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img\nAllow: /api/v1/avatar\nAllow: /api/v1/picture\n", string(body))
 }

From 00d39309811d52aee3666bb50f86469f9e0dbc09 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 01:36:54 -0500
Subject: [PATCH 05/45] image hashing to sha256

---
 backend/app/store/image/image.go      | 4 ++--
 backend/app/store/image/image_test.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 6f395c36..1026bc7d 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -2,7 +2,7 @@
 package image
 
 import (
-	"crypto/sha1"
+	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"hash/crc64"
@@ -42,7 +42,7 @@ type FileSystem struct {
 // Files partitioned across multiple subdirectories.
 func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 
-	h := sha1.Sum([]byte(name))
+	h := sha256.Sum224([]byte(name))
 	id = hex.EncodeToString(h[:])
 	if ext := path.Ext(name); ext != "" {
 		id += ext
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index fd38699a..29017910 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -24,9 +24,9 @@ func TestImage_Save(t *testing.T) {
 	}
 	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
-	assert.Equal(t, "6851dcde6024e03258a66705f29e14b506048c74.png", id)
+	assert.Equal(t, "fc77a87ad3c898b9603119711f99305145e272e103c904d85ee2deda.png", id)
 
-	dst := path.Join(loc, "02", id)
+	dst := path.Join(loc, "56", id)
 	data, err := ioutil.ReadFile(dst)
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))

From 58eb4e88523c63565b8627db24a64ccc13e19a17 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 14:09:05 -0500
Subject: [PATCH 06/45] typos

---
 backend/app/cmd/server.go         | 3 ++-
 backend/app/rest/api/rest_test.go | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 8247a900..f9a44d66 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -197,6 +197,7 @@ func (s *ServerCommand) Execute(args []string) error {
 	app, err := s.newServerApp()
 	if err != nil {
 		log.Printf("[PANIC] failed to setup application, %+v", err)
+		return err
 	}
 	if err = app.run(ctx); err != nil {
 		log.Printf("[ERROR] remark terminated with error %+v", err)
@@ -432,7 +433,7 @@ func (s *ServerCommand) makePicturesStore() (image.Interface, error) {
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
 		}
-		return &image.FileSystem{Location: s.Image.FS.Path, Partitons: s.Image.FS.Partitons, MaxSize: s.Image.MaxSize}, nil
+		return &image.FileSystem{Location: s.Image.FS.Path, Partitions: s.Image.FS.Partitons, MaxSize: s.Image.MaxSize}, nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
 }
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index c8663ac0..cd36233b 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -235,9 +235,9 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
 		ImageService: &image.FileSystem{
-			Location:  "/tmp/pics-remark42",
-			Partitons: 100,
-			MaxSize:   10000,
+			Location:   "/tmp/pics-remark42",
+			Partitions: 100,
+			MaxSize:    10000,
 		},
 
 		ImageProxy:       &proxy.Image{},

From 3c3097cf7755bcf03aee3f70e6d5188fa717dec0 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 14:09:28 -0500
Subject: [PATCH 07/45] add more image tests, generalize prep for those tests

---
 backend/app/store/image/image.go      | 13 +++--
 backend/app/store/image/image_test.go | 83 +++++++++++++++++----------
 2 files changed, 60 insertions(+), 36 deletions(-)

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 1026bc7d..7983f69e 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -1,4 +1,5 @@
-// Package image handles storing, resizing and retrival of images
+// Package image handles storing, resizing and retrieval of images
+// Provides Interface with Save and Load and one implementation on top of local file system.
 package image
 
 import (
@@ -25,9 +26,9 @@ type Interface interface {
 
 // FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
 type FileSystem struct {
-	Location  string
-	MaxSize   int
-	Partitons int
+	Location   string
+	MaxSize    int
+	Partitions int
 
 	crc struct {
 		*crc64.Table
@@ -98,13 +99,13 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 // in different subdirectories and avoid too many files in a single place.
 // the end result is a full path like this - /tmp/images/92. Number of partitions defined by FileSystem.Partitions
 func (f *FileSystem) location(id string) string {
-	if f.Partitons == 0 {
+	if f.Partitions == 0 {
 		return f.Location
 	}
 
 	f.crc.Do(func() {
 		f.crc.Table = crc64.MakeTable(crc64.ECMA)
-		p := int(math.Round(math.Log10(float64(f.Partitons))))
+		p := int(math.Round(math.Log10(float64(f.Partitions))))
 		f.crc.mask = "%0" + strconv.Itoa(p) + "d"
 		f.crc.divider = uint64(math.Pow(10, float64(p)))
 	})
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 29017910..3c24c239 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -2,6 +2,7 @@ package image
 
 import (
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path"
 	"strconv"
@@ -13,57 +14,39 @@ import (
 )
 
 func TestImage_Save(t *testing.T) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
-	require.NoError(t, err, "failed to make temp dir")
-	defer os.RemoveAll(loc)
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
 
-	svc := FileSystem{
-		Location:  loc,
-		Partitons: 100,
-		MaxSize:   50,
-	}
 	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
 	assert.Equal(t, "fc77a87ad3c898b9603119711f99305145e272e103c904d85ee2deda.png", id)
 
-	dst := path.Join(loc, "56", id)
+	dst := path.Join(svc.Location, "56", id)
 	data, err := ioutil.ReadFile(dst)
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))
 }
 
 func TestImage_SaveTooLarge(t *testing.T) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
-	require.NoError(t, err, "failed to make temp dir")
-	defer os.RemoveAll(loc)
-
-	svc := FileSystem{
-		Location:  loc,
-		Partitons: 100,
-		MaxSize:   5,
-	}
-	_, err = svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 5
+	_, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
 	assert.Error(t, err)
 	assert.EqualError(t, err, "file blah_ff1.png is too large")
 }
 
 func TestImage_Load(t *testing.T) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
-	require.NoError(t, err, "failed to make temp dir")
-	defer os.RemoveAll(loc)
 
-	// save image
-	svc := FileSystem{
-		Location:  loc,
-		Partitons: 100,
-		MaxSize:   50,
-	}
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
 	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
 
 	r, sz, err := svc.Load(id)
 	assert.NoError(t, err)
-	defer r.Close()
+	defer func() { assert.NoError(t, r.Close()) }()
 	data, err := ioutil.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))
@@ -83,12 +66,52 @@ func TestImage_location(t *testing.T) {
 		{100, "12345", "/tmp/69"},
 		{100, "xyzz", "/tmp/58"},
 		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/02"},
+		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/2"},
+		{5, "xxxyz.png", "/tmp/0"},
 		{0, "12345", "/tmp"},
 	}
 	for n, tt := range tbl {
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
-			svc := FileSystem{Location: "/tmp", Partitons: tt.partitions}
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
 			assert.Equal(t, tt.res, svc.location(tt.id))
 		})
 	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomString := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomString(rand.Intn(64))
+		parts := strings.Split(svc.location(v), "/")
+		p, err := strconv.Atoi(parts[len(parts)-1])
+		require.NoError(t, err)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	svc = FileSystem{
+		Location:   loc,
+		Partitions: 100,
+		MaxSize:    50,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+		}()
+	}
+
+	return svc, teardown
 }

From 0aba6a5653c413462a99abe31710a80652c590af Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 11 Mar 2019 14:23:12 -0500
Subject: [PATCH 08/45] longer test time for server app

---
 backend/app/cmd/server_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go
index 9e0f2309..6b56319e 100644
--- a/backend/app/cmd/server_test.go
+++ b/backend/app/cmd/server_test.go
@@ -24,7 +24,7 @@ import (
 )
 
 func TestServerApp(t *testing.T) {
-	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
+	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
 		o.Port = 18080
 		return o
 	})

From bc714480d43f9999bc07497c4dc32bd9d6d1cee4 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Tue, 19 Mar 2019 20:54:05 -0500
Subject: [PATCH 09/45] lint: multiple shadowed errors, missed comments for
 exported methods

---
 backend/app/cmd/avatar.go                 |  1 +
 backend/app/cmd/cleanup.go                |  2 +-
 backend/app/cmd/server.go                 | 16 +++++++++-------
 backend/app/migrator/disqus.go            |  4 ++--
 backend/app/migrator/native.go            | 16 ++++++++--------
 backend/app/migrator/wordpress.go         |  3 ++-
 backend/app/notify/notify.go              |  3 ++-
 backend/app/notify/telegram_test.go       |  2 ++
 backend/app/rest/api/admin.go             |  6 +++---
 backend/app/rest/api/rest_private.go      | 12 ++++++------
 backend/app/rest/api/rss.go               |  6 +++---
 backend/app/store/engine/bolt_accessor.go | 12 ++++++------
 backend/app/store/engine/bolt_admin.go    | 20 ++++++++++----------
 backend/app/store/engine/mongo.go         |  4 ++--
 backend/app/store/image/image.go          |  4 ++--
 15 files changed, 59 insertions(+), 52 deletions(-)

diff --git a/backend/app/cmd/avatar.go b/backend/app/cmd/avatar.go
index b93d6b32..5c7e9a92 100644
--- a/backend/app/cmd/avatar.go
+++ b/backend/app/cmd/avatar.go
@@ -31,6 +31,7 @@ type AvatarMigrator interface {
 
 type avatarMigrator struct{}
 
+// Migrate from one avatar store to another. Can be used to convert between stores
 func (a avatarMigrator) Migrate(dst, src avatar.Store) (int, error) {
 	return avatar.Migrate(dst, src)
 }
diff --git a/backend/app/cmd/cleanup.go b/backend/app/cmd/cleanup.go
index c7b3e3e5..bbd29417 100644
--- a/backend/app/cmd/cleanup.go
+++ b/backend/app/cmd/cleanup.go
@@ -181,7 +181,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 		Info     store.PostInfo  `json:"info,omitempty"`
 	}{}
 
-	if err := json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
+	if err = json.NewDecoder(r.Body).Decode(&commentsWithInfo); err != nil {
 		return nil, errors.Wrapf(err, "can't decode list of comments for %s", postURL)
 	}
 	return commentsWithInfo.Comments, nil
diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index f9a44d66..cef2c8e7 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -15,7 +15,7 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	auth_cache "github.com/patrickmn/go-cache"
+	authcache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth"
@@ -304,9 +304,9 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 
 	var devAuth *provider.DevAuthServer
 	if s.Auth.Dev {
-		da, err := authenticator.DevAuth()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't make dev oauth2 server")
+		da, errDevAuth := authenticator.DevAuth()
+		if errDevAuth != nil {
+			return nil, errors.Wrap(errDevAuth, "can't make dev oauth2 server")
 		}
 		devAuth = da
 	}
@@ -618,17 +618,19 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 
 // authRefreshCache used by authenticator to minimize repeatable token refreshes
 type authRefreshCache struct {
-	*auth_cache.Cache
+	*authcache.Cache
 }
 
 func newAuthRefreshCache() *authRefreshCache {
-	return &authRefreshCache{Cache: auth_cache.New(5*time.Minute, 10*time.Minute)}
+	return &authRefreshCache{Cache: authcache.New(5*time.Minute, 10*time.Minute)}
 }
 
+// Get implements cache getter with key converted to string
 func (c *authRefreshCache) Get(key interface{}) (interface{}, bool) {
 	return c.Cache.Get(key.(string))
 }
 
+// Set implements cache setter with key converted to string
 func (c *authRefreshCache) Set(key, value interface{}) {
-	c.Cache.Set(key.(string), value, auth_cache.DefaultExpiration)
+	c.Cache.Set(key.(string), value, authcache.DefaultExpiration)
 }
diff --git a/backend/app/migrator/disqus.go b/backend/app/migrator/disqus.go
index bbe0b260..14a5dc89 100644
--- a/backend/app/migrator/disqus.go
+++ b/backend/app/migrator/disqus.go
@@ -105,7 +105,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "thread" {
 					stats.inpThreads++
 					thread := disqusThread{}
-					if err := decoder.DecodeElement(&thread, &se); err != nil {
+					if err = decoder.DecodeElement(&thread, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus thread, %s", err)
 						stats.failedThreads++
 						continue
@@ -116,7 +116,7 @@ func (d *Disqus) convert(r io.Reader, siteID string) (ch chan store.Comment) {
 				if se.Name.Local == "post" {
 					stats.inpComments++
 					comment := disqusComment{}
-					if err := decoder.DecodeElement(&comment, &se); err != nil {
+					if err = decoder.DecodeElement(&comment, &se); err != nil {
 						log.Printf("[WARN] can't decode disqus comment, %s", err)
 						stats.failedPosts++
 						continue
diff --git a/backend/app/migrator/native.go b/backend/app/migrator/native.go
index 99cf9d34..40db2a17 100644
--- a/backend/app/migrator/native.go
+++ b/backend/app/migrator/native.go
@@ -15,7 +15,7 @@ import (
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
-const natvieVersion = 1
+const nativeVersion = 1
 const defaultConcurrent = 8
 
 // Native implements exporter and importer for internal store format
@@ -50,7 +50,7 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 	for i := len(topics) - 1; i >= 0; i-- { // topics from List sorted in opposite direction
 		topic := topics[i]
 		comments, e := n.DataStore.Find(store.Locator{SiteID: siteID, URL: topic.URL}, "time")
-		if err != nil {
+		if e != nil {
 			return commentsCount, e
 		}
 
@@ -75,13 +75,13 @@ func (n *Native) Export(w io.Writer, siteID string) (size int, err error) {
 
 // exportMeta appends user and post metas to exported stream
 func (n *Native) exportMeta(siteID string, w io.Writer) (err error) {
-	m := meta{Version: natvieVersion}
+	m := meta{Version: nativeVersion}
 	m.Users, m.Posts, err = n.DataStore.Metas(siteID)
 	if err != nil {
 		return errors.Wrap(err, "can't get meta")
 	}
 
-	if err := json.NewEncoder(w).Encode(m); err != nil {
+	if err = json.NewEncoder(w).Encode(m); err != nil {
 		return errors.Wrap(err, "can't encode meta")
 	}
 	return nil
@@ -96,7 +96,7 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 		return 0, errors.Wrapf(err, "failed to import meta for site %s", siteID)
 	}
 
-	if m.Version != natvieVersion && m.Version != 0 { // this version allows back compatibility with 0 version
+	if m.Version != nativeVersion && m.Version != 0 { // this version allows back compatibility with 0 version
 		return 0, errors.Errorf("unexpected import file version %d", m.Version)
 	}
 
@@ -134,9 +134,9 @@ func (n *Native) Import(reader io.Reader, siteID string) (size int, err error) {
 				log.Printf("[WARN] can't write %+v to store, %s", comment, e)
 				return
 			}
-			n := atomic.AddInt64(&comments, 1)
-			if n%1000 == 0 {
-				log.Printf("[DEBUG] imported %d comments", n)
+			num := atomic.AddInt64(&comments, 1)
+			if num%1000 == 0 {
+				log.Printf("[DEBUG] imported %d comments", num)
 			}
 		})
 
diff --git a/backend/app/migrator/wordpress.go b/backend/app/migrator/wordpress.go
index 90dce39b..c1199ea7 100644
--- a/backend/app/migrator/wordpress.go
+++ b/backend/app/migrator/wordpress.go
@@ -39,6 +39,7 @@ type wpTime struct {
 	time time.Time
 }
 
+// UnmarshalXML decoding xml with time in WP format
 func (w *wpTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
 	if err := d.DecodeElement(&v, &start); err != nil {
@@ -111,7 +112,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 				if el.Name.Local == "item" {
 					stats.inpItems++
 					item := wpItem{}
-					if err := decoder.DecodeElement(&item, &el); err != nil {
+					if err = decoder.DecodeElement(&item, &el); err != nil {
 						log.Printf("[WARN] Can't decode item, %s", err)
 						stats.failedItems++
 						continue
diff --git a/backend/app/notify/notify.go b/backend/app/notify/notify.go
index 67b1b7bb..886fabad 100644
--- a/backend/app/notify/notify.go
+++ b/backend/app/notify/notify.go
@@ -29,10 +29,11 @@ type Destination interface {
 	Send(ctx context.Context, req request) error
 }
 
-// Store defines the minimal interface accessing stored commens used by notifier
+// Store defines the minimal interface accessing stored comments used by notifier
 type Store interface {
 	Get(locator store.Locator, id string) (store.Comment, error)
 }
+
 type request struct {
 	comment store.Comment
 	parent  store.Comment
diff --git a/backend/app/notify/telegram_test.go b/backend/app/notify/telegram_test.go
index 8c429d5b..35dd2a21 100644
--- a/backend/app/notify/telegram_test.go
+++ b/backend/app/notify/telegram_test.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
 )
@@ -71,6 +72,7 @@ func TestTelegram_Send(t *testing.T) {
 	tb, err = NewTelegram("non-json-resp", "remark_test", 2*time.Second, ts.URL+"/")
 	assert.NotNil(t, err, "should failed")
 	err = tb.Send(context.TODO(), request{comment: c, parent: cp})
+	require.NotNil(t, err)
 	assert.Contains(t, err.Error(), "unexpected telegram status code 404", "send on broken tg")
 
 	assert.Equal(t, "telegram: @remark_test", tb.String())
diff --git a/backend/app/rest/api/admin.go b/backend/app/rest/api/admin.go
index 6f634e29..cba6472a 100644
--- a/backend/app/rest/api/admin.go
+++ b/backend/app/rest/api/admin.go
@@ -115,14 +115,14 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
+	if err = a.dataService.DeleteUser(claims.Audience, claims.User.ID); err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user", rest.ErrNoAccess)
 		return
 	}
 
 	if claims.User.Picture != "" && a.authenticator.AvatarProxy() != nil {
-		avatartStore := a.authenticator.AvatarProxy().Store
-		if err := avatartStore.Remove(path.Base(claims.User.Picture)); err != nil {
+		avatarStore := a.authenticator.AvatarProxy().Store
+		if err = avatarStore.Remove(path.Base(claims.User.Picture)); err != nil {
 			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user's avatar", rest.ErrInternal)
 			return
 		}
diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go
index ec17967d..97e2b493 100644
--- a/backend/app/rest/api/rest_private.go
+++ b/backend/app/rest/api/rest_private.go
@@ -228,14 +228,14 @@ func (s *Rest) userAllDataCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// get comments in 100 in each paginated request
 	for i := 0; i < 100; i++ {
-		comments, err := s.DataService.User(siteID, user.ID, 100, i*100)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get user comments", rest.ErrInternal)
+		comments, errUser := s.DataService.User(siteID, user.ID, 100, i*100)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't get user comments", rest.ErrInternal)
 			return
 		}
-		b, err := json.Marshal(comments)
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't marshal user comments", rest.ErrInternal)
+		b, errUser := json.Marshal(comments)
+		if errUser != nil {
+			rest.SendErrorJSON(w, r, http.StatusInternalServerError, errUser, "can't marshal user comments", rest.ErrInternal)
 			return
 		}
 
diff --git a/backend/app/rest/api/rss.go b/backend/app/rest/api/rss.go
index 1573befc..1b7107d2 100644
--- a/backend/app/rest/api/rss.go
+++ b/backend/app/rest/api/rss.go
@@ -57,7 +57,7 @@ func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -89,7 +89,7 @@ func (s *Rest) rssSiteCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
@@ -141,7 +141,7 @@ func (s *Rest) rssRepliesCtrl(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(data); err != nil {
+	if _, err = w.Write(data); err != nil {
 		log.Printf("[WARN] failed to send response to %s, %s", r.RemoteAddr, err)
 	}
 }
diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 9076f874..22a62893 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -155,7 +155,7 @@ func (b *BoltDB) Find(locator store.Locator, sortFld string) (comments []store.C
 
 		return bucket.ForEach(func(k, v []byte) error {
 			comment := store.Comment{}
-			if e := json.Unmarshal(v, &comment); e != nil {
+			if e = json.Unmarshal(v, &comment); e != nil {
 				return errors.Wrap(e, "failed to unmarshal")
 			}
 			comments = append(comments, comment)
@@ -195,7 +195,7 @@ func (b *BoltDB) Last(siteID string, max int) (comments []store.Comment, err err
 			}
 
 			comment := store.Comment{}
-			if e := b.load(postBkt, []byte(commentID), &comment); e != nil {
+			if e = b.load(postBkt, []byte(commentID), &comment); e != nil {
 				log.Printf("[WARN] can't load comment for %s from store %s", commentID, url)
 				continue
 			}
@@ -335,11 +335,11 @@ func (b *BoltDB) User(siteID, userID string, limit, skip int) (comments []store.
 
 	// retrieve comments for refs
 	for _, v := range commentRefs {
-		url, commentID, e := b.parseRef([]byte(v))
-		if e != nil {
-			return comments, errors.Wrapf(e, "can't parse reference %s", v)
+		url, commentID, errParse := b.parseRef([]byte(v))
+		if errParse != nil {
+			return comments, errors.Wrapf(errParse, "can't parse reference %s", v)
 		}
-		if c, e := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); e == nil {
+		if c, errRef := b.Get(store.Locator{SiteID: siteID, URL: url}, commentID); errRef == nil {
 			comments = append(comments, c)
 		}
 	}
diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go
index 579d34db..429ac991 100644
--- a/backend/app/store/engine/bolt_admin.go
+++ b/backend/app/store/engine/bolt_admin.go
@@ -29,19 +29,19 @@ func (b *BoltDB) Delete(locator store.Locator, commentID string, mode store.Dele
 		}
 
 		comment := store.Comment{}
-		if err := b.load(postBkt, []byte(commentID), &comment); err != nil {
+		if err = b.load(postBkt, []byte(commentID), &comment); err != nil {
 			return errors.Wrapf(err, "can't load key %s from bucket %s", commentID, locator.URL)
 		}
 		// set deleted status and clear fields
 		comment.SetDeleted(mode)
 
-		if err := b.save(postBkt, []byte(commentID), comment); err != nil {
+		if err = b.save(postBkt, []byte(commentID), comment); err != nil {
 			return errors.Wrapf(err, "can't save deleted comment for key %s from bucket %s", commentID, locator.URL)
 		}
 
 		// delete from "last" bucket
 		lastBkt := tx.Bucket([]byte(lastBucketName))
-		if err := lastBkt.Delete([]byte(commentID)); err != nil {
+		if err = lastBkt.Delete([]byte(commentID)); err != nil {
 			return errors.Wrapf(err, "can't delete key %s from bucket %s", commentID, lastBucketName)
 		}
 
@@ -200,8 +200,8 @@ func (b *BoltDB) IsBlocked(siteID string, userID string) (blocked bool) {
 			return nil
 		}
 
-		until, err := time.Parse(tsNano, string(val))
-		if err != nil {
+		until, e := time.Parse(tsNano, string(val))
+		if e != nil {
 			blocked = false
 			return nil
 		}
@@ -223,15 +223,15 @@ func (b *BoltDB) Blocked(siteID string) (users []store.BlockedUser, err error) {
 	err = bdb.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(blocksBucketName))
 		return bucket.ForEach(func(k []byte, v []byte) error {
-			ts, e := time.ParseInLocation(tsNano, string(v), time.Local)
-			if e != nil {
-				return errors.Wrap(e, "can't parse block ts")
+			ts, errParse := time.ParseInLocation(tsNano, string(v), time.Local)
+			if errParse != nil {
+				return errors.Wrap(errParse, "can't parse block ts")
 			}
 			if time.Now().Before(ts) {
 				// get user name from comment user section
 				userName := ""
-				userComments, e := b.User(siteID, string(k), 1, 0)
-				if e == nil && len(userComments) > 0 {
+				userComments, errUser := b.User(siteID, string(k), 1, 0)
+				if errUser == nil && len(userComments) > 0 {
 					userName = userComments[0].User.Name
 				}
 				users = append(users, store.BlockedUser{ID: string(k), Name: userName, Until: ts})
diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go
index 587e7bda..d36846d2 100644
--- a/backend/app/store/engine/mongo.go
+++ b/backend/app/store/engine/mongo.go
@@ -230,8 +230,8 @@ func (m *Mongo) Verified(siteID string) (ids []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	for _, m := range metas {
-		ids = append(ids, m.ID)
+	for _, meta := range metas {
+		ids = append(ids, meta.ID)
 	}
 	return ids, nil
 }
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 7983f69e..30a74ef4 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -51,7 +51,7 @@ func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 	location := f.location(id)
 	dst := path.Join(location, id)
 
-	if err := os.MkdirAll(location, 0700); err != nil {
+	if err = os.MkdirAll(location, 0700); err != nil {
 		return "", errors.Wrap(err, "can't make image directory")
 	}
 
@@ -64,7 +64,7 @@ func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 	if err != nil {
 		return "", errors.Wrapf(err, "can't write image file %s", dst)
 	}
-	if err := fh.Close(); err != nil {
+	if err = fh.Close(); err != nil {
 		return "", errors.Wrapf(err, "can't close image file %s", dst)
 	}
 	if written > int64(f.MaxSize) {

From eb3dd467adb232c709449be93c1762dedb1be0a5 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 22 Mar 2019 03:03:41 -0500
Subject: [PATCH 10/45] change image location to user based, make random uuid
 for file name

---
 backend/app/rest/api/rest.go              |  2 +-
 backend/app/rest/api/rest_private.go      |  5 +-
 backend/app/rest/api/rest_private_test.go |  5 +-
 backend/app/rest/api/rest_public.go       |  4 +-
 backend/app/store/image/image.go          | 70 +++++++++++++----------
 backend/app/store/image/image_test.go     | 48 ++++++++--------
 6 files changed, 74 insertions(+), 60 deletions(-)

diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index e09a458e..c7349f84 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -221,7 +221,7 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
-			ropen.Get("/picture/{id}", s.loadPictureCtrl)
+			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 
 			ropen.Mount("/rss", s.rssRoutes())
 			ropen.Mount("/img", s.ImageProxy.Routes())
diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go
index 97e2b493..b7063656 100644
--- a/backend/app/rest/api/rest_private.go
+++ b/backend/app/rest/api/rest_private.go
@@ -301,14 +301,13 @@ func (s *Rest) savePictureCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = file.Close() }()
 
-	picName := fmt.Sprintf("%s_%d_%s", user.ID, time.Now().Nanosecond(), header.Filename)
-	id, err := s.ImageService.Save(picName, file)
+	id, err := s.ImageService.Save(header.Filename, user.ID, file)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't save image", rest.ErrInternal)
 		return
 	}
 
-	render.JSON(w, r, R.JSON{"location": id})
+	render.JSON(w, r, R.JSON{"id": id})
 }
 
 func (s *Rest) isReadOnly(locator store.Locator) bool {
diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index 46a92ad6..cf053e2a 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -509,6 +509,7 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+	require.NoError(t, err)
 	req.Header.Add("Content-Type", contentType)
 	req.Header.Add("X-JWT", devToken)
 	resp, err := client.Do(req)
@@ -519,10 +520,10 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 
 	m := map[string]string{}
 	err = json.Unmarshal(body, &m)
-	assert.Contains(t, m["location"], ".png")
+	assert.Contains(t, m["id"], ".png")
 
 	// load picture
-	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, m["location"]))
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, m["id"]))
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err = ioutil.ReadAll(resp.Body)
diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go
index 6aae3850..3713ef53 100644
--- a/backend/app/rest/api/rest_public.go
+++ b/backend/app/rest/api/rest_public.go
@@ -332,7 +332,7 @@ func (s *Rest) listCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GET /picture/{id} - get picture
+// GET /picture/{user}/{id} - get picture
 func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 
 	imgContentType := func(img string) string {
@@ -348,7 +348,7 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		return "image/*"
 	}
 
-	id := chi.URLParam(r, "id")
+	id := chi.URLParam(r, "user") + "/" + chi.URLParam(r, "id")
 	imgRdr, size, err := s.ImageService.Load(id)
 	if err != nil {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+id, rest.ErrAssetNotFound)
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 30a74ef4..2fa45bbe 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -3,25 +3,26 @@
 package image
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"hash/crc64"
 	"io"
 	"math"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
 // Interface defines Save and Load methods
 type Interface interface {
-	Save(name string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
-	Load(id string) (io.ReadCloser, int64, error)         // load image by ID. Caller has to close the reader.
+	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
 }
 
 // FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
@@ -41,17 +42,17 @@ type FileSystem struct {
 // Save data from reader for given file name to local FS. Returns id as a hash of name
 // name should be passed in unique prefix, for example with userID_*
 // Files partitioned across multiple subdirectories.
-func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
 
-	h := sha256.Sum224([]byte(name))
-	id = hex.EncodeToString(h[:])
-	if ext := path.Ext(name); ext != "" {
-		id += ext
+	uid, err := uuid.NewUUID()
+	if err != nil {
+		return "", errors.Wrap(err, "can't make image uuid")
 	}
-	location := f.location(id)
-	dst := path.Join(location, id)
 
-	if err = os.MkdirAll(location, 0700); err != nil {
+	id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(id)
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
 		return "", errors.Wrap(err, "can't make image directory")
 	}
 
@@ -71,17 +72,16 @@ func (f *FileSystem) Save(name string, r io.Reader) (id string, err error) {
 		if err = os.Remove(dst); err != nil {
 			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
 		}
-		return "", errors.Errorf("file %s is too large", name)
+		return "", errors.Errorf("file %s is too large", fileName)
 	}
-	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), name)
+	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName)
 	return id, nil
 }
 
 // Load image from FS. Uses id to get partition subdirectory.
 // returns ReadCloser and caller should call close after processing completed.
 func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
-	location := f.location(id)
-	imgFile := path.Join(location, id)
+	imgFile := f.location(id)
 
 	st, err := os.Stat(imgFile)
 	if err != nil {
@@ -95,21 +95,33 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 	return fh, st.Size(), nil
 }
 
-// get location (directory) for id by adding partition to the final path in order to keep files
+// get location (full path) for id by adding partition to the final path in order to keep files
 // in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/images/92. Number of partitions defined by FileSystem.Partitions
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png. Number of partitions defined by FileSystem.
+// Partitions
 func (f *FileSystem) location(id string) string {
-	if f.Partitions == 0 {
-		return f.Location
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
 	}
 
-	f.crc.Do(func() {
-		f.crc.Table = crc64.MakeTable(crc64.ECMA)
-		p := int(math.Round(math.Log10(float64(f.Partitions))))
-		f.crc.mask = "%0" + strconv.Itoa(p) + "d"
-		f.crc.divider = uint64(math.Pow(10, float64(p)))
-	})
-	checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
-	partition := checksum64 % f.crc.divider
-	return path.Join(f.Location, fmt.Sprintf(f.crc.mask, partition))
+	user := "unknown"
+	file := id
+	elems := strings.Split(id, "/")
+	if len(elems) == 2 {
+		user, file = elems[0], elems[1]
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(f.Location, user, file)
+	}
+	return path.Join(f.Location, user, partition(id), file)
 }
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 3c24c239..ef99fc05 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -4,7 +4,6 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
-	"path"
 	"strconv"
 	"strings"
 	"testing"
@@ -17,12 +16,13 @@ func TestImage_Save(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
-	assert.Equal(t, "fc77a87ad3c898b9603119711f99305145e272e103c904d85ee2deda.png", id)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
 
-	dst := path.Join(svc.Location, "56", id)
-	data, err := ioutil.ReadFile(dst)
+	data, err := ioutil.ReadFile(svc.location(id))
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))
 }
@@ -31,9 +31,9 @@ func TestImage_SaveTooLarge(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 	svc.MaxSize = 5
-	_, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	_, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah"))
 	assert.Error(t, err)
-	assert.EqualError(t, err, "file blah_ff1.png is too large")
+	assert.Contains(t, err.Error(), "is too large")
 }
 
 func TestImage_Load(t *testing.T) {
@@ -41,8 +41,9 @@ func TestImage_Load(t *testing.T) {
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
 
-	id, err := svc.Save("blah_ff1.png", strings.NewReader("blah blah"))
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
 	assert.NoError(t, err)
+	t.Log(id)
 
 	r, sz, err := svc.Load(id)
 	assert.NoError(t, err)
@@ -60,15 +61,15 @@ func TestImage_location(t *testing.T) {
 		partitions int
 		id, res    string
 	}{
-		{10, "abcdefg", "/tmp/2"},
-		{10, "abcdefe", "/tmp/1"},
-		{10, "12345", "/tmp/9"},
-		{100, "12345", "/tmp/69"},
-		{100, "xyzz", "/tmp/58"},
-		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/02"},
-		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/2"},
-		{5, "xxxyz.png", "/tmp/0"},
-		{0, "12345", "/tmp"},
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "abcdefe", "/tmp/unknown/1/abcdefe"},
+		{10, "12345", "/tmp/unknown/9/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/02/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/2/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "xxxyz.png", "/tmp/unknown/0/xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
 	}
 	for n, tt := range tbl {
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
@@ -79,20 +80,21 @@ func TestImage_location(t *testing.T) {
 
 	// generate random names and make sure partition never runs out of allowed
 	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-	randomString := func(n int) string {
+	randomID := func(n int) string {
 		b := make([]rune, n)
 		for i := range b {
 			b[i] = letterRunes[rand.Intn(len(letterRunes))]
 		}
-		return string(b)
+		return "user1" + "/" + string(b)
 	}
 
 	svc := FileSystem{Location: "/tmp", Partitions: 10}
 	for i := 0; i < 1000; i++ {
-		v := randomString(rand.Intn(64))
-		parts := strings.Split(svc.location(v), "/")
-		p, err := strconv.Atoi(parts[len(parts)-1])
-		require.NoError(t, err)
+		v := randomID(rand.Intn(64))
+		location := svc.location(v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
 		assert.True(t, p >= 0 && p < 10)
 	}
 }

From debd914e39a22b6a3907b6d348c98833b0a3ee3f Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 22 Mar 2019 03:12:54 -0500
Subject: [PATCH 11/45] adjust image comments

---
 backend/app/store/image/image.go | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 2fa45bbe..8c6c42da 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -39,9 +39,8 @@ type FileSystem struct {
 	}
 }
 
-// Save data from reader for given file name to local FS. Returns id as a hash of name
-// name should be passed in unique prefix, for example with userID_*
-// Files partitioned across multiple subdirectories.
+// Save data from reader for given file name to local FS. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
 func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
 
 	uid, err := uuid.NewUUID()
@@ -95,10 +94,10 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 	return fh, st.Size(), nil
 }
 
-// get location (full path) for id by adding partition to the final path in order to keep files
-// in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png. Number of partitions defined by FileSystem.
-// Partitions
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
 func (f *FileSystem) location(id string) string {
 
 	partition := func(id string) string {
@@ -113,15 +112,14 @@ func (f *FileSystem) location(id string) string {
 		return fmt.Sprintf(f.crc.mask, partition)
 	}
 
-	user := "unknown"
-	file := id
-	elems := strings.Split(id, "/")
-	if len(elems) == 2 {
-		user, file = elems[0], elems[1]
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
 	}
 
 	if f.Partitions == 0 {
-		return path.Join(f.Location, user, file)
+		return path.Join(f.Location, user, file) // avoid partition directory if 0 Partitions
 	}
+
 	return path.Join(f.Location, user, partition(id), file)
 }

From eb79c3d9f90554a95b8272b9ccf0310ac6856720 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 02:56:58 -0500
Subject: [PATCH 12/45] implement two-stage image commit with background
 cleanup

---
 backend/app/store/image/image.go              | 76 +++++++++++++--
 backend/app/store/image/image_test.go         | 97 ++++++++++++++++++-
 backend/go.sum                                |  1 -
 .../github.com/hashicorp/errwrap/README.md    |  2 +-
 .../github.com/hashicorp/errwrap/go.mod       |  1 +
 .../github.com/hashicorp/golang-lru/lru.go    | 30 +++---
 .../shurcooL/sanitized_anchor_name/go.mod     |  1 +
 backend/vendor/golang.org/x/time/rate/rate.go | 24 ++---
 .../golang.org/x/time/rate/rate_go16.go       | 21 ----
 .../golang.org/x/time/rate/rate_go17.go       | 21 ----
 backend/vendor/modules.txt                    |  8 +-
 11 files changed, 189 insertions(+), 93 deletions(-)
 create mode 100644 backend/vendor/github.com/hashicorp/errwrap/go.mod
 create mode 100644 backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod
 delete mode 100644 backend/vendor/golang.org/x/time/rate/rate_go16.go
 delete mode 100644 backend/vendor/golang.org/x/time/rate/rate_go17.go

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 8c6c42da..75b043cb 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -3,6 +3,7 @@
 package image
 
 import (
+	"context"
 	"fmt"
 	"hash/crc64"
 	"io"
@@ -13,6 +14,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
@@ -28,8 +30,10 @@ type Interface interface {
 // FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
 type FileSystem struct {
 	Location   string
+	Staging    string
 	MaxSize    int
 	Partitions int
+	TTL        time.Duration // for how long file allowed on staging
 
 	crc struct {
 		*crc64.Table
@@ -39,7 +43,7 @@ type FileSystem struct {
 	}
 }
 
-// Save data from reader for given file name to local FS. Returns id as user/uuid.ext
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
 // Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
 func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
 
@@ -49,7 +53,7 @@ func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id strin
 	}
 
 	id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
-	dst := f.location(id)
+	dst := f.location(f.Staging, id)
 
 	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
 		return "", errors.Wrap(err, "can't make image directory")
@@ -77,14 +81,36 @@ func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id strin
 	return id, nil
 }
 
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
 // Load image from FS. Uses id to get partition subdirectory.
 // returns ReadCloser and caller should call close after processing completed.
 func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
-	imgFile := f.location(id)
 
-	st, err := os.Stat(imgFile)
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
 	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't get image size for %s", id)
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
 	}
 
 	fh, err := os.Open(imgFile)
@@ -94,11 +120,45 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 	return fh, st.Size(), nil
 }
 
+// Cleanup runs periodic scan of staging and removes old files based on TTL
+func (f *FileSystem) Cleanup(ctx context.Context) {
+
+	cleanup := func() {
+		err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			age := time.Since(info.ModTime())
+			if age > f.TTL {
+				log.Printf("[INFO] remove staging image %s, age %v", path, age)
+				return os.Remove(path)
+			}
+			return nil
+		})
+		if err != nil {
+			log.Printf("[WARN] failed to cleanup images, %v", err)
+		}
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(f.TTL / 2):
+			cleanup()
+		}
+	}
+}
+
 // location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
 // and avoid too many files in a single place.
 // the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
 // Number of partitions defined by FileSystem.Partitions
-func (f *FileSystem) location(id string) string {
+func (f *FileSystem) location(base string, id string) string {
 
 	partition := func(id string) string {
 		f.crc.Do(func() {
@@ -118,8 +178,8 @@ func (f *FileSystem) location(id string) string {
 	}
 
 	if f.Partitions == 0 {
-		return path.Join(f.Location, user, file) // avoid partition directory if 0 Partitions
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
 	}
 
-	return path.Join(f.Location, user, partition(id), file)
+	return path.Join(base, user, partition(id), file)
 }
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index ef99fc05..0179df1d 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -1,12 +1,14 @@
 package image
 
 import (
+	"context"
 	"io/ioutil"
 	"math/rand"
 	"os"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -22,7 +24,29 @@ func TestImage_Save(t *testing.T) {
 	assert.Contains(t, id, ".png")
 	t.Log(id)
 
-	data, err := ioutil.ReadFile(svc.location(id))
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestImage_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
 	assert.NoError(t, err)
 	assert.Equal(t, "blah blah", string(data))
 }
@@ -36,7 +60,7 @@ func TestImage_SaveTooLarge(t *testing.T) {
 	assert.Contains(t, err.Error(), "is too large")
 }
 
-func TestImage_Load(t *testing.T) {
+func TestImage_LoadAfterSave(t *testing.T) {
 
 	svc, teardown := prepareImageTest(t)
 	defer teardown()
@@ -56,6 +80,28 @@ func TestImage_Load(t *testing.T) {
 	assert.NotNil(t, err)
 }
 
+func TestImage_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
 func TestImage_location(t *testing.T) {
 	tbl := []struct {
 		partitions int
@@ -74,7 +120,7 @@ func TestImage_location(t *testing.T) {
 	for n, tt := range tbl {
 		t.Run(strconv.Itoa(n), func(t *testing.T) {
 			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
-			assert.Equal(t, tt.res, svc.location(tt.id))
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
 		})
 	}
 
@@ -91,7 +137,7 @@ func TestImage_location(t *testing.T) {
 	svc := FileSystem{Location: "/tmp", Partitions: 10}
 	for i := 0; i < 1000; i++ {
 		v := randomID(rand.Intn(64))
-		location := svc.location(v)
+		location := svc.location("/tmp", v)
 		elems := strings.Split(location, "/")
 		p, err := strconv.Atoi(elems[3])
 		require.NoError(t, err, location)
@@ -99,12 +145,54 @@ func TestImage_location(t *testing.T) {
 	}
 }
 
+func TestImage_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string, content string) (path string) {
+		id, err := svc.Save(file, user, strings.NewReader(content))
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		require.Equal(t, content, string(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1", "blah blah1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1", "blah blah2")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2", "blah blah3")
+
+	svc.TTL = time.Millisecond * 300
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(1000 * time.Millisecond)
+		cancel()
+	}()
+
+	svc.Cleanup(ctx)
+
+	_, err := os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
 func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
 	loc, err := ioutil.TempDir("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
 
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
 	svc = FileSystem{
 		Location:   loc,
+		Staging:    staging,
 		Partitions: 100,
 		MaxSize:    50,
 	}
@@ -112,6 +200,7 @@ func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
 	teardown = func() {
 		defer func() {
 			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
 		}()
 	}
 
diff --git a/backend/go.sum b/backend/go.sum
index f7ad9620..97fddbf8 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -81,7 +81,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
diff --git a/backend/vendor/github.com/hashicorp/errwrap/README.md b/backend/vendor/github.com/hashicorp/errwrap/README.md
index 1c95f597..444df08f 100644
--- a/backend/vendor/github.com/hashicorp/errwrap/README.md
+++ b/backend/vendor/github.com/hashicorp/errwrap/README.md
@@ -48,7 +48,7 @@ func main() {
 	// We can use the Contains helpers to check if an error contains
 	// another error. It is safe to do this with a nil error, or with
 	// an error that doesn't even use the errwrap package.
-	if errwrap.Contains(err, ErrNotExist) {
+	if errwrap.Contains(err, "does not exist") {
 		// Do something
 	}
 	if errwrap.ContainsType(err, new(os.PathError)) {
diff --git a/backend/vendor/github.com/hashicorp/errwrap/go.mod b/backend/vendor/github.com/hashicorp/errwrap/go.mod
new file mode 100644
index 00000000..c9b84022
--- /dev/null
+++ b/backend/vendor/github.com/hashicorp/errwrap/go.mod
@@ -0,0 +1 @@
+module github.com/hashicorp/errwrap
diff --git a/backend/vendor/github.com/hashicorp/golang-lru/lru.go b/backend/vendor/github.com/hashicorp/golang-lru/lru.go
index 1cbe04b7..c8d9b0a2 100644
--- a/backend/vendor/github.com/hashicorp/golang-lru/lru.go
+++ b/backend/vendor/github.com/hashicorp/golang-lru/lru.go
@@ -40,35 +40,31 @@ func (c *Cache) Purge() {
 // Add adds a value to the cache.  Returns true if an eviction occurred.
 func (c *Cache) Add(key, value interface{}) (evicted bool) {
 	c.lock.Lock()
-	evicted = c.lru.Add(key, value)
-	c.lock.Unlock()
-	return evicted
+	defer c.lock.Unlock()
+	return c.lru.Add(key, value)
 }
 
 // Get looks up a key's value from the cache.
 func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
 	c.lock.Lock()
-	value, ok = c.lru.Get(key)
-	c.lock.Unlock()
-	return value, ok
+	defer c.lock.Unlock()
+	return c.lru.Get(key)
 }
 
 // Contains checks if a key is in the cache, without updating the
 // recent-ness or deleting it for being stale.
 func (c *Cache) Contains(key interface{}) bool {
 	c.lock.RLock()
-	containKey := c.lru.Contains(key)
-	c.lock.RUnlock()
-	return containKey
+	defer c.lock.RUnlock()
+	return c.lru.Contains(key)
 }
 
 // Peek returns the key value (or undefined if not found) without updating
 // the "recently used"-ness of the key.
 func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) {
 	c.lock.RLock()
-	value, ok = c.lru.Peek(key)
-	c.lock.RUnlock()
-	return value, ok
+	defer c.lock.RUnlock()
+	return c.lru.Peek(key)
 }
 
 // ContainsOrAdd checks if a key is in the cache  without updating the
@@ -102,15 +98,13 @@ func (c *Cache) RemoveOldest() {
 // Keys returns a slice of the keys in the cache, from oldest to newest.
 func (c *Cache) Keys() []interface{} {
 	c.lock.RLock()
-	keys := c.lru.Keys()
-	c.lock.RUnlock()
-	return keys
+	defer c.lock.RUnlock()
+	return c.lru.Keys()
 }
 
 // Len returns the number of items in the cache.
 func (c *Cache) Len() int {
 	c.lock.RLock()
-	length := c.lru.Len()
-	c.lock.RUnlock()
-	return length
+	defer c.lock.RUnlock()
+	return c.lru.Len()
 }
diff --git a/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod b/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod
new file mode 100644
index 00000000..1e255347
--- /dev/null
+++ b/backend/vendor/github.com/shurcooL/sanitized_anchor_name/go.mod
@@ -0,0 +1 @@
+module github.com/shurcooL/sanitized_anchor_name
diff --git a/backend/vendor/golang.org/x/time/rate/rate.go b/backend/vendor/golang.org/x/time/rate/rate.go
index eabcd114..ae93e247 100644
--- a/backend/vendor/golang.org/x/time/rate/rate.go
+++ b/backend/vendor/golang.org/x/time/rate/rate.go
@@ -6,6 +6,7 @@
 package rate
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"sync"
@@ -212,19 +213,8 @@ func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation {
 	return &r
 }
 
-// contextContext is a temporary(?) copy of the context.Context type
-// to support both Go 1.6 using golang.org/x/net/context and Go 1.7+
-// with the built-in context package. If people ever stop using Go 1.6
-// we can remove this.
-type contextContext interface {
-	Deadline() (deadline time.Time, ok bool)
-	Done() <-chan struct{}
-	Err() error
-	Value(key interface{}) interface{}
-}
-
 // Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) wait(ctx contextContext) (err error) {
+func (lim *Limiter) Wait(ctx context.Context) (err error) {
 	return lim.WaitN(ctx, 1)
 }
 
@@ -232,7 +222,7 @@ func (lim *Limiter) wait(ctx contextContext) (err error) {
 // It returns an error if n exceeds the Limiter's burst size, the Context is
 // canceled, or the expected wait time exceeds the Context's Deadline.
 // The burst limit is ignored if the rate limit is Inf.
-func (lim *Limiter) waitN(ctx contextContext, n int) (err error) {
+func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
 	if n > lim.burst && lim.limit != Inf {
 		return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, lim.burst)
 	}
@@ -253,8 +243,12 @@ func (lim *Limiter) waitN(ctx contextContext, n int) (err error) {
 	if !r.ok {
 		return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n)
 	}
-	// Wait
-	t := time.NewTimer(r.DelayFrom(now))
+	// Wait if necessary
+	delay := r.DelayFrom(now)
+	if delay == 0 {
+		return nil
+	}
+	t := time.NewTimer(delay)
 	defer t.Stop()
 	select {
 	case <-t.C:
diff --git a/backend/vendor/golang.org/x/time/rate/rate_go16.go b/backend/vendor/golang.org/x/time/rate/rate_go16.go
deleted file mode 100644
index 6bab1850..00000000
--- a/backend/vendor/golang.org/x/time/rate/rate_go16.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !go1.7
-
-package rate
-
-import "golang.org/x/net/context"
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.waitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	return lim.waitN(ctx, n)
-}
diff --git a/backend/vendor/golang.org/x/time/rate/rate_go17.go b/backend/vendor/golang.org/x/time/rate/rate_go17.go
deleted file mode 100644
index f90d85f5..00000000
--- a/backend/vendor/golang.org/x/time/rate/rate_go17.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.7
-
-package rate
-
-import "context"
-
-// Wait is shorthand for WaitN(ctx, 1).
-func (lim *Limiter) Wait(ctx context.Context) (err error) {
-	return lim.waitN(ctx, 1)
-}
-
-// WaitN blocks until lim permits n events to happen.
-// It returns an error if n exceeds the Limiter's burst size, the Context is
-// canceled, or the expected wait time exceeds the Context's Deadline.
-func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) {
-	return lim.waitN(ctx, n)
-}
diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt
index 3f1038ff..d4dad4b1 100644
--- a/backend/vendor/modules.txt
+++ b/backend/vendor/modules.txt
@@ -58,11 +58,11 @@ github.com/golang/protobuf/proto
 github.com/google/uuid
 # github.com/gorilla/feeds v1.1.0
 github.com/gorilla/feeds
-# github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce
+# github.com/hashicorp/errwrap v1.0.0
 github.com/hashicorp/errwrap
 # github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
 github.com/hashicorp/go-multierror
-# github.com/hashicorp/golang-lru v0.5.1
+# github.com/hashicorp/golang-lru v0.5.0
 github.com/hashicorp/golang-lru
 github.com/hashicorp/golang-lru/simplelru
 # github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
@@ -79,7 +79,7 @@ github.com/pkg/errors
 github.com/pmezard/go-difflib/difflib
 # github.com/rakyll/statik v0.1.3
 github.com/rakyll/statik/fs
-# github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95
+# github.com/shurcooL/sanitized_anchor_name v1.0.0
 github.com/shurcooL/sanitized_anchor_name
 # github.com/stretchr/testify v1.3.0
 github.com/stretchr/testify/assert
@@ -106,7 +106,7 @@ golang.org/x/oauth2/jws
 golang.org/x/oauth2/jwt
 # golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb
 golang.org/x/sys/unix
-# golang.org/x/time v0.0.0-20170927054726-6dc17368e09b
+# golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 golang.org/x/time/rate
 # google.golang.org/appengine v1.4.0
 google.golang.org/appengine

From b49e242891736426ea3b2efcaebd2dfd2f2087f5 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 03:06:24 -0500
Subject: [PATCH 13/45] wire image staging and image cleanup

---
 backend/app/cmd/server.go        | 18 +++++++++++++++---
 backend/app/store/image/image.go |  3 +++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index cef2c8e7..3180f0a8 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -99,8 +99,9 @@ type StoreGroup struct {
 type ImageGroup struct {
 	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
 	FS   struct {
-		Path      string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
-		Partitons int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
+		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
+		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
+		Partitions int    `long:"partitions" env:"PARTITIONS" default:"100" description:"partitions (subdirs)"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
 	Bolt struct {
 		File string `long:"file" env:"FILE" default:"./var/pictures.db" description:"images bolt file location"`
@@ -177,6 +178,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
+	imageService  image.Interface
 	terminated    chan struct{}
 }
 
@@ -320,6 +322,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
+		imageService:  pictStore,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -347,10 +350,14 @@ func (a *serverApp) run(ctx context.Context) error {
 		a.notifyService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
+
 	a.activateBackup(ctx) // runs in goroutine for each site
 	if a.Auth.Dev {
 		go a.devAuth.Run(context.Background()) // dev oauth2 server on :8084
 	}
+
+	go a.imageService.Cleanup(ctx) // pictures cleanup for staging images
+
 	a.restSrv.Run(a.Port)
 	close(a.terminated)
 	return nil
@@ -433,7 +440,12 @@ func (s *ServerCommand) makePicturesStore() (image.Interface, error) {
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
 		}
-		return &image.FileSystem{Location: s.Image.FS.Path, Partitions: s.Image.FS.Partitons, MaxSize: s.Image.MaxSize}, nil
+		return &image.FileSystem{
+			Location:   s.Image.FS.Path,
+			Staging:    s.Image.FS.Staging,
+			Partitions: s.Image.FS.Partitions,
+			MaxSize:    s.Image.MaxSize,
+		}, nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
 }
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 75b043cb..d9e58a98 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -24,7 +24,9 @@ import (
 // Interface defines Save and Load methods
 type Interface interface {
 	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
+	Commit(id string) error                                                  // move image from staging to permanent
 	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
+	Cleanup(ctx context.Context)                                             // run removal loop for old images on staging
 }
 
 // FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
@@ -122,6 +124,7 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 
 // Cleanup runs periodic scan of staging and removes old files based on TTL
 func (f *FileSystem) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", f.TTL)
 
 	cleanup := func() {
 		err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {

From de292a4146b86057edc0d94b51097a854c4fa25c Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 03:47:07 -0500
Subject: [PATCH 14/45] add extraction of image ids

---
 backend/app/store/image/image.go      | 24 ++++++++++++++++++++++++
 backend/app/store/image/image_test.go | 10 ++++++++++
 2 files changed, 34 insertions(+)

diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index d9e58a98..16fa144d 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/PuerkitoBio/goquery"
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
@@ -186,3 +187,26 @@ func (f *FileSystem) location(base string, id string) string {
 
 	return path.Join(base, user, partition(id), file)
 }
+
+// ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
+func ExtractPictures(commentHTML string, match string) (ids []string, err error) {
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create document")
+	}
+	result := []string{}
+	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+		if im, ok := s.Attr("src"); ok {
+			if strings.Contains(im, match) {
+				elems := strings.Split(im, "/")
+				if len(elems) >= 2 {
+					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
+					result = append(result, id)
+				}
+			}
+		}
+	})
+
+	return result, nil
+}
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 0179df1d..ef3ec135 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -183,6 +183,16 @@ func TestImage_Cleanup(t *testing.T) {
 	assert.NotNil(t, err, "no file on staging anymore")
 }
 
+func TestExtractPictures(t *testing.T) {
+	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+<img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
+	ids, err := ExtractPictures(html, "/blah/")
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(ids), "two images")
+	assert.Equal(t, "user1/pic1.png", ids[0])
+	assert.Equal(t, "user2/pic3.png", ids[1])
+}
+
 func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
 	loc, err := ioutil.TempDir("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")

From c990b05c211c4cdd18f3d013febbab7615e0b3bb Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 14:54:45 -0500
Subject: [PATCH 15/45] add image service with delayed commit. Move cleanup
 loop to service

---
 backend/app/cmd/server.go                |   4 +-
 backend/app/rest/api/rest.go             |   2 +-
 backend/app/store/image/fs_store.go      | 161 +++++++++++++++++
 backend/app/store/image/fs_store_test.go | 212 +++++++++++++++++++++++
 backend/app/store/image/image.go         | 201 ++++-----------------
 backend/app/store/image/image_mock.go    |  92 ++++++++++
 backend/app/store/image/image_test.go    | 209 ++--------------------
 backend/app/store/service/service.go     |   8 +-
 backend/go.mod                           |   1 +
 backend/go.sum                           |   3 +
 backend/vendor/modules.txt               |   2 +
 11 files changed, 526 insertions(+), 369 deletions(-)
 create mode 100644 backend/app/store/image/fs_store.go
 create mode 100644 backend/app/store/image/fs_store_test.go
 create mode 100644 backend/app/store/image/image_mock.go

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 3180f0a8..f5292400 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -178,7 +178,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
-	imageService  image.Interface
+	imageService  image.Store
 	terminated    chan struct{}
 }
 
@@ -434,7 +434,7 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
-func (s *ServerCommand) makePicturesStore() (image.Interface, error) {
+func (s *ServerCommand) makePicturesStore() (image.Store, error) {
 	switch s.Image.Type {
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index c7349f84..8625d504 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -45,7 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
-	ImageService     image.Interface
+	ImageService     image.Store
 
 	WebRoot         string
 	RemarkURL       string
diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go
new file mode 100644
index 00000000..ae6c4c64
--- /dev/null
+++ b/backend/app/store/image/fs_store.go
@@ -0,0 +1,161 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"hash/crc64"
+	"io"
+	"log"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+// FileSystem provides image Store for local files. Saves and loads files from Location, restricts max size.
+type FileSystem struct {
+	Location   string
+	Staging    string
+	MaxSize    int
+	Partitions int
+
+	crc struct {
+		*crc64.Table
+		sync.Once
+		mask    string
+		divider uint64
+	}
+}
+
+// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
+// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
+func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
+
+	uid, err := uuid.NewUUID()
+	if err != nil {
+		return "", errors.Wrap(err, "can't make image uuid")
+	}
+
+	id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
+	dst := f.location(f.Staging, id)
+
+	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
+		return "", errors.Wrap(err, "can't make image directory")
+	}
+
+	fh, err := os.Create(dst)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't make image file %s", dst)
+	}
+	lr := io.LimitReader(r, int64(f.MaxSize)+1)
+	written, err := io.Copy(fh, lr)
+	if err != nil {
+		return "", errors.Wrapf(err, "can't write image file %s", dst)
+	}
+	if err = fh.Close(); err != nil {
+		return "", errors.Wrapf(err, "can't close image file %s", dst)
+	}
+	if written > int64(f.MaxSize) {
+		if err = os.Remove(dst); err != nil {
+			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
+		}
+		return "", errors.Errorf("file %s is too large", fileName)
+	}
+	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName)
+	return id, nil
+}
+
+// Commit file stored in staging location by moving it to permanent location
+func (f *FileSystem) Commit(id string) error {
+	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
+
+	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
+		return errors.Wrap(err, "can't make image directory")
+	}
+
+	err := os.Rename(stagingImage, permImage)
+	return errors.Wrapf(err, "failed to commit image %s", id)
+}
+
+// Load image from FS. Uses id to get partition subdirectory.
+// returns ReadCloser and caller should call close after processing completed.
+func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
+
+	// get image file by id. first try permanent location and if not found - staging
+	img := func(id string) (file string, st os.FileInfo, err error) {
+		file = f.location(f.Location, id)
+		st, err = os.Stat(file)
+		if err != nil {
+			file = f.location(f.Staging, id)
+			st, err = os.Stat(file)
+		}
+		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
+	}
+
+	imgFile, st, err := img(id)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
+	}
+
+	fh, err := os.Open(imgFile)
+	if err != nil {
+		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
+	}
+	return fh, st.Size(), nil
+}
+
+// Cleanup runs scan of staging and removes old files based on ttl
+func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		age := time.Since(info.ModTime())
+		if age > ttl {
+			log.Printf("[INFO] remove staging image %s, age %v", path, age)
+			return os.Remove(path)
+		}
+		return nil
+	})
+	return errors.Wrap(err, "failed to cleanup images")
+}
+
+// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
+// and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
+// Number of partitions defined by FileSystem.Partitions
+func (f *FileSystem) location(base string, id string) string {
+
+	partition := func(id string) string {
+		f.crc.Do(func() {
+			f.crc.Table = crc64.MakeTable(crc64.ECMA)
+			p := int(math.Round(math.Log10(float64(f.Partitions))))
+			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
+			f.crc.divider = uint64(math.Pow(10, float64(p)))
+		})
+		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
+		partition := checksum64 % f.crc.divider
+		return fmt.Sprintf(f.crc.mask, partition)
+	}
+
+	user, file := "unknown", id // default if no user in id
+	if elems := strings.Split(id, "/"); len(elems) == 2 {
+		user, file = elems[0], elems[1] // user in id
+	}
+
+	if f.Partitions == 0 {
+		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
+	}
+
+	return path.Join(base, user, partition(id), file)
+}
diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go
new file mode 100644
index 00000000..5906d814
--- /dev/null
+++ b/backend/app/store/image/fs_store_test.go
@@ -0,0 +1,212 @@
+package image
+
+import (
+	"context"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFsStore_Save(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	assert.Contains(t, id, "user1/")
+	assert.Contains(t, id, ".png")
+	t.Log(id)
+
+	img := svc.location(svc.Staging, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestFsStore_SaveAndCommit(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
+	require.NoError(t, err)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	imgStaging := svc.location(svc.Staging, id)
+	_, err = os.Stat(imgStaging)
+	assert.NotNil(t, err, "no file on staging anymore")
+
+	img := svc.location(svc.Location, id)
+	t.Log(img)
+	data, err := ioutil.ReadFile(img)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+}
+
+func TestFsStore_SaveTooLarge(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+	svc.MaxSize = 5
+	_, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah"))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is too large")
+}
+
+func TestFsStore_LoadAfterSave(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	t.Log(id)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_LoadAfterCommit(t *testing.T) {
+
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
+	assert.NoError(t, err)
+	t.Log(id)
+	err = svc.Commit(id)
+	require.NoError(t, err)
+
+	r, sz, err := svc.Load(id)
+	assert.NoError(t, err)
+	defer func() { assert.NoError(t, r.Close()) }()
+	data, err := ioutil.ReadAll(r)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah blah", string(data))
+	assert.Equal(t, int64(9), sz)
+	_, _, err = svc.Load("abcd")
+	assert.NotNil(t, err)
+}
+
+func TestFsStore_location(t *testing.T) {
+	tbl := []struct {
+		partitions int
+		id, res    string
+	}{
+		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
+		{10, "abcdefe", "/tmp/unknown/1/abcdefe"},
+		{10, "12345", "/tmp/unknown/9/12345"},
+		{100, "12345", "/tmp/unknown/69/12345"},
+		{100, "xyzz", "/tmp/unknown/58/xyzz"},
+		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/02/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/2/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "xxxyz.png", "/tmp/unknown/0/xxxyz.png"},
+		{0, "12345", "/tmp/unknown/12345"},
+	}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
+			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
+		})
+	}
+
+	// generate random names and make sure partition never runs out of allowed
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	randomID := func(n int) string {
+		b := make([]rune, n)
+		for i := range b {
+			b[i] = letterRunes[rand.Intn(len(letterRunes))]
+		}
+		return "user1" + "/" + string(b)
+	}
+
+	svc := FileSystem{Location: "/tmp", Partitions: 10}
+	for i := 0; i < 1000; i++ {
+		v := randomID(rand.Intn(64))
+		location := svc.location("/tmp", v)
+		elems := strings.Split(location, "/")
+		p, err := strconv.Atoi(elems[3])
+		require.NoError(t, err, location)
+		assert.True(t, p >= 0 && p < 10)
+	}
+}
+
+func TestFsStore_Cleanup(t *testing.T) {
+	svc, teardown := prepareImageTest(t)
+	defer teardown()
+
+	save := func(file string, user string, content string) (path string) {
+		id, err := svc.Save(file, user, strings.NewReader(content))
+		require.NoError(t, err)
+		img := svc.location(svc.Staging, id)
+		data, err := ioutil.ReadFile(img)
+		require.NoError(t, err)
+		require.Equal(t, content, string(data))
+		return img
+	}
+
+	// save 3 images to staging
+	img1 := save("blah_ff1.png", "user1", "blah blah1")
+	time.Sleep(100 * time.Millisecond)
+	img2 := save("blah_ff2.png", "user1", "blah blah2")
+	time.Sleep(100 * time.Millisecond)
+	img3 := save("blah_ff3.png", "user2", "blah blah3")
+
+	time.Sleep(100 * time.Millisecond) // make first image expired
+	err := svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img1)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img2)
+	assert.NoError(t, err, "file on staging")
+	_, err = os.Stat(img3)
+	assert.NoError(t, err, "file on staging")
+
+	time.Sleep(200 * time.Millisecond) // make all images expired
+	err = svc.Cleanup(context.Background(), time.Millisecond*300)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(img2)
+	assert.NotNil(t, err, "no file on staging anymore")
+	_, err = os.Stat(img3)
+	assert.NotNil(t, err, "no file on staging anymore")
+}
+
+func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
+	loc, err := ioutil.TempDir("", "test_image_r42")
+	require.NoError(t, err, "failed to make temp dir")
+
+	staging, err := ioutil.TempDir("", "test_image_r42.staging")
+	require.NoError(t, err, "failed to make temp staging dir")
+
+	svc = FileSystem{
+		Location:   loc,
+		Staging:    staging,
+		Partitions: 100,
+		MaxSize:    50,
+	}
+
+	teardown = func() {
+		defer func() {
+			assert.NoError(t, os.RemoveAll(loc))
+			assert.NoError(t, os.RemoveAll(staging))
+		}()
+	}
+
+	return svc, teardown
+}
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 16fa144d..e0c95224 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -1,195 +1,49 @@
 // Package image handles storing, resizing and retrieval of images
-// Provides Interface with Save and Load and one implementation on top of local file system.
+// Provides Store with Save and Load and one implementation on top of local file system.
+// Service object encloses Store and add common methods, this is the one consumer should use
 package image
 
+//go:generate sh -c "mockgen -source=image.go -package=image > image_mock.go"
+
 import (
 	"context"
-	"fmt"
-	"hash/crc64"
 	"io"
-	"math"
-	"os"
-	"path"
-	"path/filepath"
-	"strconv"
+	"log"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	log "github.com/go-pkgz/lgr"
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
-// Interface defines Save and Load methods
-type Interface interface {
+// Store defines interface for saving and loading pictures.
+// Declares two-stage save with commit
+type Store interface {
 	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
 	Commit(id string) error                                                  // move image from staging to permanent
 	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
 	Cleanup(ctx context.Context)                                             // run removal loop for old images on staging
 }
 
-// FileSystem provides image Interface for local files. Saves and loads files from Location, restricts max size
-type FileSystem struct {
-	Location   string
-	Staging    string
-	MaxSize    int
-	Partitions int
-	TTL        time.Duration // for how long file allowed on staging
-
-	crc struct {
-		*crc64.Table
-		sync.Once
-		mask    string
-		divider uint64
-	}
+// Service extends Store with common functions needed for any store implementation
+type Service struct {
+	Store
+	TTL time.Duration // for how long file allowed on staging
 }
 
-// Save data from reader for given file name to local FS, staging directory. Returns id as user/uuid.ext
-// Files partitioned across multiple subdirectories and the final path includes part, i.e. /location/user1/03/123-4567.png
-func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id string, err error) {
-
-	uid, err := uuid.NewUUID()
-	if err != nil {
-		return "", errors.Wrap(err, "can't make image uuid")
-	}
-
-	id = path.Join(userID, uid.String()) + filepath.Ext(fileName) // make id as user/uuid.ext
-	dst := f.location(f.Staging, id)
-
-	if err = os.MkdirAll(path.Dir(dst), 0700); err != nil {
-		return "", errors.Wrap(err, "can't make image directory")
-	}
-
-	fh, err := os.Create(dst)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't make image file %s", dst)
-	}
-	lr := io.LimitReader(r, int64(f.MaxSize)+1)
-	written, err := io.Copy(fh, lr)
-	if err != nil {
-		return "", errors.Wrapf(err, "can't write image file %s", dst)
-	}
-	if err = fh.Close(); err != nil {
-		return "", errors.Wrapf(err, "can't close image file %s", dst)
-	}
-	if written > int64(f.MaxSize) {
-		if err = os.Remove(dst); err != nil {
-			log.Printf("[WARN] can't remove image file %s, %v", dst, err)
-		}
-		return "", errors.Errorf("file %s is too large", fileName)
-	}
-	log.Printf("[DEBUG] file %s saved for image %s", fh.Name(), fileName)
-	return id, nil
-}
-
-// Commit file stored in staging location by moving it to permanent location
-func (f *FileSystem) Commit(id string) error {
-	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
-
-	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
-		return errors.Wrap(err, "can't make image directory")
-	}
-
-	err := os.Rename(stagingImage, permImage)
-	return errors.Wrapf(err, "failed to commit image %s", id)
-}
-
-// Load image from FS. Uses id to get partition subdirectory.
-// returns ReadCloser and caller should call close after processing completed.
-func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
-
-	// get image file by id. first try permanent location and if not found - staging
-	img := func(id string) (file string, st os.FileInfo, err error) {
-		file = f.location(f.Location, id)
-		st, err = os.Stat(file)
-		if err != nil {
-			file = f.location(f.Staging, id)
-			st, err = os.Stat(file)
-		}
-		return file, st, errors.Wrapf(err, "can't get image stats for %s", id)
-	}
-
-	imgFile, st, err := img(id)
-	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't get image file for %s", id)
-	}
-
-	fh, err := os.Open(imgFile)
-	if err != nil {
-		return nil, 0, errors.Wrapf(err, "can't load image %s", id)
-	}
-	return fh, st.Size(), nil
-}
-
-// Cleanup runs periodic scan of staging and removes old files based on TTL
-func (f *FileSystem) Cleanup(ctx context.Context) {
-	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", f.TTL)
-
-	cleanup := func() {
-		err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
+// Submit multiple ids for delayed commit
+func (s *Service) Submit(ids []string, delay time.Duration) {
+	time.AfterFunc(delay, func() {
+		for _, id := range ids {
+			if err := s.Commit(id); err != nil {
+				log.Printf("[WARN] failed to commit image %s", id)
 			}
-			if info.IsDir() {
-				return nil
-			}
-			age := time.Since(info.ModTime())
-			if age > f.TTL {
-				log.Printf("[INFO] remove staging image %s, age %v", path, age)
-				return os.Remove(path)
-			}
-			return nil
-		})
-		if err != nil {
-			log.Printf("[WARN] failed to cleanup images, %v", err)
 		}
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
-			return
-		case <-time.After(f.TTL / 2):
-			cleanup()
-		}
-	}
-}
-
-// location gets full path for id by adding partition to the final path in order to keep files in different subdirectories
-// and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/images/user1/92/xxx-yyy.png.
-// Number of partitions defined by FileSystem.Partitions
-func (f *FileSystem) location(base string, id string) string {
-
-	partition := func(id string) string {
-		f.crc.Do(func() {
-			f.crc.Table = crc64.MakeTable(crc64.ECMA)
-			p := int(math.Round(math.Log10(float64(f.Partitions))))
-			f.crc.mask = "%0" + strconv.Itoa(p) + "d"
-			f.crc.divider = uint64(math.Pow(10, float64(p)))
-		})
-		checksum64 := crc64.Checksum([]byte(id), f.crc.Table)
-		partition := checksum64 % f.crc.divider
-		return fmt.Sprintf(f.crc.mask, partition)
-	}
-
-	user, file := "unknown", id // default if no user in id
-	if elems := strings.Split(id, "/"); len(elems) == 2 {
-		user, file = elems[0], elems[1] // user in id
-	}
-
-	if f.Partitions == 0 {
-		return path.Join(base, user, file) // avoid partition directory if 0 Partitions
-	}
-
-	return path.Join(base, user, partition(id), file)
+	})
 }
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
-func ExtractPictures(commentHTML string, match string) (ids []string, err error) {
+func (s *Service) ExtractPictures(commentHTML string, match string) (ids []string, err error) {
 
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
@@ -210,3 +64,18 @@ func ExtractPictures(commentHTML string, match string) (ids []string, err error)
 
 	return result, nil
 }
+
+// Cleanup runs periodic cleanup with TTL. Blocking loop, should be called inside of goroutine by consumer
+func (s *Service) Cleanup(ctx context.Context) {
+	log.Printf("[INFO] start pictures cleanup, staging ttl=%v", s.TTL)
+
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
+			return
+		case <-time.After(s.TTL / 2):
+			s.Store.Cleanup(ctx)
+		}
+	}
+}
diff --git a/backend/app/store/image/image_mock.go b/backend/app/store/image/image_mock.go
new file mode 100644
index 00000000..5dc04453
--- /dev/null
+++ b/backend/app/store/image/image_mock.go
@@ -0,0 +1,92 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: image.go
+
+// Package image is a generated GoMock package.
+package image
+
+import (
+	context "context"
+	gomock "github.com/golang/mock/gomock"
+	io "io"
+	reflect "reflect"
+)
+
+// MockStore is a mock of Store interface
+type MockStore struct {
+	ctrl     *gomock.Controller
+	recorder *MockStoreMockRecorder
+}
+
+// MockStoreMockRecorder is the mock recorder for MockStore
+type MockStoreMockRecorder struct {
+	mock *MockStore
+}
+
+// NewMockStore creates a new mock instance
+func NewMockStore(ctrl *gomock.Controller) *MockStore {
+	mock := &MockStore{ctrl: ctrl}
+	mock.recorder = &MockStoreMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockStore) EXPECT() *MockStoreMockRecorder {
+	return m.recorder
+}
+
+// Save mocks base method
+func (m *MockStore) Save(fileName, userID string, r io.Reader) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Save", fileName, userID, r)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Save indicates an expected call of Save
+func (mr *MockStoreMockRecorder) Save(fileName, userID, r interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockStore)(nil).Save), fileName, userID, r)
+}
+
+// Commit mocks base method
+func (m *MockStore) Commit(id string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Commit", id)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Commit indicates an expected call of Commit
+func (mr *MockStoreMockRecorder) Commit(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockStore)(nil).Commit), id)
+}
+
+// Load mocks base method
+func (m *MockStore) Load(id string) (io.ReadCloser, int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Load", id)
+	ret0, _ := ret[0].(io.ReadCloser)
+	ret1, _ := ret[1].(int64)
+	ret2, _ := ret[2].(error)
+	return ret0, ret1, ret2
+}
+
+// Load indicates an expected call of Load
+func (mr *MockStoreMockRecorder) Load(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockStore)(nil).Load), id)
+}
+
+// Cleanup mocks base method
+func (m *MockStore) Cleanup(ctx context.Context) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Cleanup", ctx)
+}
+
+// Cleanup indicates an expected call of Cleanup
+func (mr *MockStoreMockRecorder) Cleanup(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockStore)(nil).Cleanup), ctx)
+}
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index ef3ec135..44ced538 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -2,217 +2,34 @@ package image
 
 import (
 	"context"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
 
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestImage_Save(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
-	assert.NoError(t, err)
-	assert.Contains(t, id, "user1/")
-	assert.Contains(t, id, ".png")
-	t.Log(id)
-
-	img := svc.location(svc.Staging, id)
-	t.Log(img)
-	data, err := ioutil.ReadFile(img)
-	assert.NoError(t, err)
-	assert.Equal(t, "blah blah", string(data))
-}
-
-func TestImage_SaveAndCommit(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	id, err := svc.Save("file1.png", "user1", strings.NewReader("blah blah"))
-	require.NoError(t, err)
-	err = svc.Commit(id)
-	require.NoError(t, err)
-
-	imgStaging := svc.location(svc.Staging, id)
-	_, err = os.Stat(imgStaging)
-	assert.NotNil(t, err, "no file on staging anymore")
-
-	img := svc.location(svc.Location, id)
-	t.Log(img)
-	data, err := ioutil.ReadFile(img)
-	assert.NoError(t, err)
-	assert.Equal(t, "blah blah", string(data))
-}
-
-func TestImage_SaveTooLarge(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-	svc.MaxSize = 5
-	_, err := svc.Save("blah_ff1.png", "user2", strings.NewReader("blah blah"))
-	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "is too large")
-}
-
-func TestImage_LoadAfterSave(t *testing.T) {
-
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
-	assert.NoError(t, err)
-	t.Log(id)
-
-	r, sz, err := svc.Load(id)
-	assert.NoError(t, err)
-	defer func() { assert.NoError(t, r.Close()) }()
-	data, err := ioutil.ReadAll(r)
-	assert.NoError(t, err)
-	assert.Equal(t, "blah blah", string(data))
-	assert.Equal(t, int64(9), sz)
-	_, _, err = svc.Load("abcd")
-	assert.NotNil(t, err)
-}
-
-func TestImage_LoadAfterCommit(t *testing.T) {
-
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	id, err := svc.Save("blah_ff1.png", "user1", strings.NewReader("blah blah"))
-	assert.NoError(t, err)
-	t.Log(id)
-	err = svc.Commit(id)
-	require.NoError(t, err)
-
-	r, sz, err := svc.Load(id)
-	assert.NoError(t, err)
-	defer func() { assert.NoError(t, r.Close()) }()
-	data, err := ioutil.ReadAll(r)
-	assert.NoError(t, err)
-	assert.Equal(t, "blah blah", string(data))
-	assert.Equal(t, int64(9), sz)
-	_, _, err = svc.Load("abcd")
-	assert.NotNil(t, err)
-}
-
-func TestImage_location(t *testing.T) {
-	tbl := []struct {
-		partitions int
-		id, res    string
-	}{
-		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
-		{10, "abcdefe", "/tmp/unknown/1/abcdefe"},
-		{10, "12345", "/tmp/unknown/9/12345"},
-		{100, "12345", "/tmp/unknown/69/12345"},
-		{100, "xyzz", "/tmp/unknown/58/xyzz"},
-		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/02/6851dcde6024e03258a66705f29e14b506048c74.png"},
-		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/2/6851dcde6024e03258a66705f29e14b506048c74.png"},
-		{5, "xxxyz.png", "/tmp/unknown/0/xxxyz.png"},
-		{0, "12345", "/tmp/unknown/12345"},
-	}
-	for n, tt := range tbl {
-		t.Run(strconv.Itoa(n), func(t *testing.T) {
-			svc := FileSystem{Location: "/tmp", Partitions: tt.partitions}
-			assert.Equal(t, tt.res, svc.location("/tmp", tt.id))
-		})
-	}
-
-	// generate random names and make sure partition never runs out of allowed
-	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-	randomID := func(n int) string {
-		b := make([]rune, n)
-		for i := range b {
-			b[i] = letterRunes[rand.Intn(len(letterRunes))]
-		}
-		return "user1" + "/" + string(b)
-	}
-
-	svc := FileSystem{Location: "/tmp", Partitions: 10}
-	for i := 0; i < 1000; i++ {
-		v := randomID(rand.Intn(64))
-		location := svc.location("/tmp", v)
-		elems := strings.Split(location, "/")
-		p, err := strconv.Atoi(elems[3])
-		require.NoError(t, err, location)
-		assert.True(t, p >= 0 && p < 10)
-	}
-}
-
-func TestImage_Cleanup(t *testing.T) {
-	svc, teardown := prepareImageTest(t)
-	defer teardown()
-
-	save := func(file string, user string, content string) (path string) {
-		id, err := svc.Save(file, user, strings.NewReader(content))
-		require.NoError(t, err)
-		img := svc.location(svc.Staging, id)
-		data, err := ioutil.ReadFile(img)
-		require.NoError(t, err)
-		require.Equal(t, content, string(data))
-		return img
-	}
-
-	// save 3 images to staging
-	img1 := save("blah_ff1.png", "user1", "blah blah1")
-	time.Sleep(100 * time.Millisecond)
-	img2 := save("blah_ff2.png", "user1", "blah blah2")
-	time.Sleep(100 * time.Millisecond)
-	img3 := save("blah_ff3.png", "user2", "blah blah3")
-
-	svc.TTL = time.Millisecond * 300
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		time.Sleep(1000 * time.Millisecond)
-		cancel()
-	}()
-
-	svc.Cleanup(ctx)
-
-	_, err := os.Stat(img1)
-	assert.NotNil(t, err, "no file on staging anymore")
-	_, err = os.Stat(img2)
-	assert.NotNil(t, err, "no file on staging anymore")
-	_, err = os.Stat(img3)
-	assert.NotNil(t, err, "no file on staging anymore")
-}
-
-func TestExtractPictures(t *testing.T) {
+func TestService_ExtractPictures(t *testing.T) {
+	svc := Service{}
 	html := `blah <img src="/blah/user1/pic1.png"/> foo 
 <img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
-	ids, err := ExtractPictures(html, "/blah/")
+	ids, err := svc.ExtractPictures(html, "/blah/")
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(ids), "two images")
 	assert.Equal(t, "user1/pic1.png", ids[0])
 	assert.Equal(t, "user2/pic3.png", ids[1])
 }
 
-func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
-	loc, err := ioutil.TempDir("", "test_image_r42")
-	require.NoError(t, err, "failed to make temp dir")
+func TestService_Cleanup(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
 
-	staging, err := ioutil.TempDir("", "test_image_r42.staging")
-	require.NoError(t, err, "failed to make temp staging dir")
+	store := NewMockStore(ctrl)
+	store.EXPECT().Cleanup(gomock.Any()).Times(10)
 
-	svc = FileSystem{
-		Location:   loc,
-		Staging:    staging,
-		Partitions: 100,
-		MaxSize:    50,
-	}
-
-	teardown = func() {
-		defer func() {
-			assert.NoError(t, os.RemoveAll(loc))
-			assert.NoError(t, os.RemoveAll(staging))
-		}()
-	}
-
-	return svc, teardown
+	svc := Service{Store: store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*550)
+	defer cancel()
+	svc.Cleanup(ctx)
 }
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 4878dae9..9bc355e2 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -129,8 +129,8 @@ func (s *DataStore) SetPin(locator store.Locator, commentID string, status bool)
 // Vote for comment by id and locator
 func (s *DataStore) Vote(locator store.Locator, commentID string, userID string, val bool) (comment store.Comment, err error) {
 
-	cLock := s.getsScopedLocks(locator.URL) // get lock for URL scope
-	cLock.Lock()                            // prevents race on voting
+	cLock := s.getScopedLocks(locator.URL) // get lock for URL scope
+	cLock.Lock()                           // prevents race on voting
 	defer cLock.Unlock()
 
 	comment, err = s.Get(locator, commentID)
@@ -455,8 +455,8 @@ func (s *DataStore) upsAndDowns(c store.Comment) (ups, downs int) {
 	return ups, downs
 }
 
-// getsScopedLocks pull lock from the map if found or create a new one
-func (s *DataStore) getsScopedLocks(id string) (lock sync.Locker) {
+// getScopedLocks pull lock from the map if found or create a new one
+func (s *DataStore) getScopedLocks(id string) (lock sync.Locker) {
 	s.scopedLocks.Do(func() { s.scopedLocks.locks = map[string]sync.Locker{} })
 
 	s.scopedLocks.Lock()
diff --git a/backend/go.mod b/backend/go.mod
index 65c95d3d..55c2b513 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -20,6 +20,7 @@ require (
 	github.com/go-pkgz/repeater v1.1.1
 	github.com/go-pkgz/rest v1.4.0
 	github.com/go-pkgz/syncs v1.1.0
+	github.com/golang/mock v1.2.0
 	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
 	github.com/gorilla/feeds v1.1.0
 	github.com/hashicorp/errwrap v1.0.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 97fddbf8..b7de4812 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -43,6 +43,8 @@ github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
 github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
 github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
 github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
@@ -78,6 +80,7 @@ github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt
index d4dad4b1..2a3a7886 100644
--- a/backend/vendor/modules.txt
+++ b/backend/vendor/modules.txt
@@ -52,6 +52,8 @@ github.com/go-pkgz/rest
 github.com/go-pkgz/rest/logger
 # github.com/go-pkgz/syncs v1.1.0
 github.com/go-pkgz/syncs
+# github.com/golang/mock v1.2.0
+github.com/golang/mock/gomock
 # github.com/golang/protobuf v1.2.0
 github.com/golang/protobuf/proto
 # github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c

From 58ec50e61348c3e4ccae455c3f80d16ef2c77646 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 18:33:19 -0500
Subject: [PATCH 16/45] wire image submit to store.Service

---
 backend/app/cmd/server.go             | 32 +++++++++++++++------------
 backend/app/rest/api/rest.go          |  2 +-
 backend/app/store/image/image.go      | 25 +++++++++++++--------
 backend/app/store/image/image_mock.go | 11 +++++----
 backend/app/store/image/image_test.go | 32 ++++++++++++++++++++++++---
 backend/app/store/service/service.go  |  7 ++++++
 6 files changed, 78 insertions(+), 31 deletions(-)

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index f5292400..8fa4ef8a 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -178,7 +178,7 @@ type serverApp struct {
 	dataService   *service.DataStore
 	avatarStore   avatar.Store
 	notifyService *notify.Service
-	imageService  image.Store
+	imageService  *image.Service
 	terminated    chan struct{}
 }
 
@@ -232,6 +232,11 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		return nil, errors.Wrap(err, "failed to make admin store")
 	}
 
+	imageService, err := s.makePicturesStore()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to make pictures store")
+	}
+
 	dataService := &service.DataStore{
 		Interface:              storeEngine,
 		EditDuration:           s.EditDuration,
@@ -239,6 +244,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		MaxCommentSize:         s.MaxCommentSize,
 		MaxVotes:               s.MaxVotes,
 		PositiveScore:          s.PositiveScore,
+		ImageService:           imageService,
 		TitleExtractor:         service.NewTitleExtractor(http.Client{Timeout: time.Second * 5}),
 		RestrictedWordsMatcher: service.NewRestrictedWordsMatcher(service.StaticRestrictedWordsLister{Words: s.RestrictedWords}),
 	}
@@ -274,11 +280,6 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	imgProxy := &proxy.Image{Enabled: s.ImageProxy, RoutePath: "/api/v1/img", RemarkURL: s.RemarkURL}
 	commentFormatter := store.NewCommentFormatter(imgProxy)
 
-	pictStore, err := s.makePicturesStore()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make pictures store")
-	}
-
 	sslConfig, err := s.makeSSLConfig()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make config of ssl server params")
@@ -299,7 +300,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		NotifyService:    notifyService,
 		SSLConfig:        sslConfig,
 		UpdateLimiter:    s.UpdateLimit,
-		ImageService:     pictStore,
+		ImageService:     imageService,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore
@@ -322,7 +323,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 		dataService:   dataService,
 		avatarStore:   avatarStore,
 		notifyService: notifyService,
-		imageService:  pictStore,
+		imageService:  imageService,
 		terminated:    make(chan struct{}),
 	}, nil
 }
@@ -434,17 +435,20 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 	return nil, errors.Errorf("unsupported avatar store type %s", s.Avatar.Type)
 }
 
-func (s *ServerCommand) makePicturesStore() (image.Store, error) {
+func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 	switch s.Image.Type {
 	case "fs":
 		if err := makeDirs(s.Image.FS.Path); err != nil {
 			return nil, err
 		}
-		return &image.FileSystem{
-			Location:   s.Image.FS.Path,
-			Staging:    s.Image.FS.Staging,
-			Partitions: s.Image.FS.Partitions,
-			MaxSize:    s.Image.MaxSize,
+		return &image.Service{
+			Store: &image.FileSystem{
+				Location:   s.Image.FS.Path,
+				Staging:    s.Image.FS.Staging,
+				Partitions: s.Image.FS.Partitions,
+				MaxSize:    s.Image.MaxSize,
+			},
+			TTL: s.EditDuration + time.Second, // add extra second to image TTL for staging
 		}, nil
 	}
 	return nil, errors.Errorf("unsupported pictures store type %s", s.Image.Type)
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index 8625d504..7844217e 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -45,7 +45,7 @@ type Rest struct {
 	CommentFormatter *store.CommentFormatter
 	Migrator         *Migrator
 	NotifyService    *notify.Service
-	ImageService     image.Store
+	ImageService     *image.Service
 
 	WebRoot         string
 	RemarkURL       string
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index e0c95224..08097f00 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -22,18 +22,23 @@ type Store interface {
 	Save(fileName string, userID string, r io.Reader) (id string, err error) // get name and reader and returns ID of stored image
 	Commit(id string) error                                                  // move image from staging to permanent
 	Load(id string) (io.ReadCloser, int64, error)                            // load image by ID. Caller has to close the reader.
-	Cleanup(ctx context.Context)                                             // run removal loop for old images on staging
+	Cleanup(ctx context.Context, ttl time.Duration) error                    // run removal loop for old images on staging
 }
 
 // Service extends Store with common functions needed for any store implementation
 type Service struct {
 	Store
-	TTL time.Duration // for how long file allowed on staging
+	TTL      time.Duration // for how long file allowed on staging
+	ImageAPI string        // image api matching path
 }
 
 // Submit multiple ids for delayed commit
-func (s *Service) Submit(ids []string, delay time.Duration) {
-	time.AfterFunc(delay, func() {
+func (s *Service) Submit(ids []string) {
+	if len(ids) == 0 {
+		return
+	}
+
+	time.AfterFunc(s.TTL, func() {
 		for _, id := range ids {
 			if err := s.Commit(id); err != nil {
 				log.Printf("[WARN] failed to commit image %s", id)
@@ -43,16 +48,16 @@ func (s *Service) Submit(ids []string, delay time.Duration) {
 }
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
-func (s *Service) ExtractPictures(commentHTML string, match string) (ids []string, err error) {
+func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error) {
 
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML))
 	if err != nil {
 		return nil, errors.Wrap(err, "can't create document")
 	}
 	result := []string{}
-	doc.Find("img").Each(func(i int, s *goquery.Selection) {
-		if im, ok := s.Attr("src"); ok {
-			if strings.Contains(im, match) {
+	doc.Find("img").Each(func(i int, sl *goquery.Selection) {
+		if im, ok := sl.Attr("src"); ok {
+			if strings.Contains(im, s.ImageAPI) {
 				elems := strings.Split(im, "/")
 				if len(elems) >= 2 {
 					id := elems[len(elems)-2] + "/" + elems[len(elems)-1]
@@ -75,7 +80,9 @@ func (s *Service) Cleanup(ctx context.Context) {
 			log.Printf("[INFO] cleanup terminated, %v", ctx.Err())
 			return
 		case <-time.After(s.TTL / 2):
-			s.Store.Cleanup(ctx)
+			if err := s.Store.Cleanup(ctx, s.TTL); err != nil {
+				log.Printf("[WARN] failed to cleanup, %v", err)
+			}
 		}
 	}
 }
diff --git a/backend/app/store/image/image_mock.go b/backend/app/store/image/image_mock.go
index 5dc04453..0049e558 100644
--- a/backend/app/store/image/image_mock.go
+++ b/backend/app/store/image/image_mock.go
@@ -9,6 +9,7 @@ import (
 	gomock "github.com/golang/mock/gomock"
 	io "io"
 	reflect "reflect"
+	time "time"
 )
 
 // MockStore is a mock of Store interface
@@ -80,13 +81,15 @@ func (mr *MockStoreMockRecorder) Load(id interface{}) *gomock.Call {
 }
 
 // Cleanup mocks base method
-func (m *MockStore) Cleanup(ctx context.Context) {
+func (m *MockStore) Cleanup(ctx context.Context, ttl time.Duration) error {
 	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "Cleanup", ctx)
+	ret := m.ctrl.Call(m, "Cleanup", ctx, ttl)
+	ret0, _ := ret[0].(error)
+	return ret0
 }
 
 // Cleanup indicates an expected call of Cleanup
-func (mr *MockStoreMockRecorder) Cleanup(ctx interface{}) *gomock.Call {
+func (mr *MockStoreMockRecorder) Cleanup(ctx, ttl interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockStore)(nil).Cleanup), ctx)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockStore)(nil).Cleanup), ctx, ttl)
 }
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 44ced538..ad12dae5 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -11,22 +11,48 @@ import (
 )
 
 func TestService_ExtractPictures(t *testing.T) {
-	svc := Service{}
+	svc := Service{ImageAPI: "/blah/"}
 	html := `blah <img src="/blah/user1/pic1.png"/> foo 
 <img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/>`
-	ids, err := svc.ExtractPictures(html, "/blah/")
+	ids, err := svc.ExtractPictures(html)
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(ids), "two images")
 	assert.Equal(t, "user1/pic1.png", ids[0])
 	assert.Equal(t, "user2/pic3.png", ids[1])
 }
 
+func TestService_Submit(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	store := NewMockStore(ctrl)
+
+	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit([]string{"id1", "id2", "id3"})
+	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(nil)
+	time.Sleep(time.Millisecond * 500)
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	store := NewMockStore(ctrl)
+
+	store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
+	svc.Submit([]string{"id1", "id2", "id3"})
+	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
+	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(nil)
+}
+
 func TestService_Cleanup(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
 	store := NewMockStore(ctrl)
-	store.EXPECT().Cleanup(gomock.Any()).Times(10)
+	store.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Times(10)
 
 	svc := Service{Store: store, TTL: 100 * time.Millisecond}
 	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*550)
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 9bc355e2..d65f5cbc 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -12,6 +12,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -28,6 +29,7 @@ type DataStore struct {
 	PositiveScore          bool
 	TitleExtractor         *TitleExtractor
 	RestrictedWordsMatcher *RestrictedWordsMatcher
+	ImageService           *image.Service
 
 	// granular locks
 	scopedLocks struct {
@@ -90,6 +92,11 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
+	imgIds, err := s.ImageService.ExtractPictures(comment.Text)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to prepare extract pictures")
+	}
+	s.ImageService.Submit(imgIds) // submit images commit, delayed by EditDuration
 	return s.Interface.Create(comment)
 }
 

From db3f22d9fd9a4ea5319bbd96044df4342bc6d2e0 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 19:13:58 -0500
Subject: [PATCH 17/45] change image submit to single goroutine with active
 wait. flush all submitted on close

---
 backend/app/cmd/server.go             |  1 +
 backend/app/rest/api/rest_test.go     | 12 ++++---
 backend/app/store/image/image.go      | 48 ++++++++++++++++++++++++---
 backend/app/store/image/image_test.go | 44 ++++++++++++++++--------
 4 files changed, 81 insertions(+), 24 deletions(-)

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 8fa4ef8a..5b88d957 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -349,6 +349,7 @@ func (a *serverApp) run(ctx context.Context) error {
 			log.Printf("[WARN] failed to close avatar store, %s", e)
 		}
 		a.notifyService.Close()
+		a.imageService.Close()
 		log.Print("[INFO] shutdown completed")
 	}()
 
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index cd36233b..ef5108d7 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -234,12 +234,14 @@ func startupT(t *testing.T) (ts *httptest.Server, srv *Rest, teardown func()) {
 		Cache:     memCache,
 		WebRoot:   "/tmp",
 		RemarkURL: "https://demo.remark42.com",
-		ImageService: &image.FileSystem{
-			Location:   "/tmp/pics-remark42",
-			Partitions: 100,
-			MaxSize:    10000,
+		ImageService: &image.Service{
+			Store: &image.FileSystem{
+				Location:   "/tmp/pics-remark42",
+				Partitions: 100,
+				MaxSize:    10000,
+			},
+			TTL: time.Millisecond * 100,
 		},
-
 		ImageProxy:       &proxy.Image{},
 		ReadOnlyAge:      10,
 		CommentFormatter: store.NewCommentFormatter(&proxy.Image{}),
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 08097f00..8c5b620f 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -10,6 +10,8 @@ import (
 	"io"
 	"log"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -30,6 +32,18 @@ type Service struct {
 	Store
 	TTL      time.Duration // for how long file allowed on staging
 	ImageAPI string        // image api matching path
+
+	wg       sync.WaitGroup
+	submitCh chan submitReq
+	once     sync.Once
+	term     int32
+}
+
+const submitQueueSize = 5000
+
+type submitReq struct {
+	ID string
+	TS time.Time
 }
 
 // Submit multiple ids for delayed commit
@@ -38,13 +52,27 @@ func (s *Service) Submit(ids []string) {
 		return
 	}
 
-	time.AfterFunc(s.TTL, func() {
-		for _, id := range ids {
-			if err := s.Commit(id); err != nil {
-				log.Printf("[WARN] failed to commit image %s", id)
+	s.once.Do(func() {
+		s.submitCh = make(chan submitReq, submitQueueSize)
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			for req := range s.submitCh {
+				// wait for TTL expiration with emergency pass on term
+				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
+					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
+				}
+				if err := s.Commit(req.ID); err != nil {
+					log.Printf("[WARN] failed to commit image %s", req.ID)
+				}
 			}
-		}
+			log.Printf("[INFO] image submiter terminated")
+		}()
 	})
+
+	for _, id := range ids {
+		s.submitCh <- submitReq{ID: id, TS: time.Now()}
+	}
 }
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
@@ -86,3 +114,13 @@ func (s *Service) Cleanup(ctx context.Context) {
 		}
 	}
 }
+
+// Close flushes all in-progress submits and enforces waiting commits
+func (s *Service) Close() {
+	log.Printf("[INFO] close image service ")
+	atomic.AddInt32(&s.term, 1) // enforce non-delayed commits for all ids left in submitCh
+	if s.submitCh != nil {
+		close(s.submitCh)
+	}
+	s.wg.Wait()
+}
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index ad12dae5..335327fa 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -21,6 +21,19 @@ func TestService_ExtractPictures(t *testing.T) {
 	assert.Equal(t, "user2/pic3.png", ids[1])
 }
 
+func TestService_Cleanup(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	store := NewMockStore(ctrl)
+	store.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Times(10)
+
+	svc := Service{Store: store, TTL: 100 * time.Millisecond}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*549)
+	defer cancel()
+	svc.Cleanup(ctx)
+}
+
 func TestService_Submit(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -34,11 +47,27 @@ func TestService_Submit(t *testing.T) {
 	time.Sleep(time.Millisecond * 500)
 }
 
-func TestService_SubmitDelay(t *testing.T) {
+func TestService_Close(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 	store := NewMockStore(ctrl)
 
+	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
+	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
+	svc.Submit([]string{"id1", "id2", "id3"})
+	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(nil)
+	svc.Close()
+}
+
+func TestService_SubmitDelay(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer func() {
+		ctrl.Finish()
+	}()
+
+	store := NewMockStore(ctrl)
+
 	store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
 	svc.Submit([]string{"id1", "id2", "id3"})
@@ -46,16 +75,3 @@ func TestService_SubmitDelay(t *testing.T) {
 	svc.Submit([]string{"id4", "id5"})
 	svc.Submit(nil)
 }
-
-func TestService_Cleanup(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	store := NewMockStore(ctrl)
-	store.EXPECT().Cleanup(gomock.Any(), gomock.Any()).Times(10)
-
-	svc := Service{Store: store, TTL: 100 * time.Millisecond}
-	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*550)
-	defer cancel()
-	svc.Cleanup(ctx)
-}

From 42767b94a4b2d5819280f839aa34aa2349142e9d Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 19:15:40 -0500
Subject: [PATCH 18/45] missed vendor for gomock

---
 backend/vendor/github.com/golang/mock/AUTHORS |  12 +
 .../github.com/golang/mock/CONTRIBUTORS       |  37 ++
 backend/vendor/github.com/golang/mock/LICENSE | 202 +++++++++
 .../github.com/golang/mock/gomock/call.go     | 420 ++++++++++++++++++
 .../github.com/golang/mock/gomock/callset.go  | 108 +++++
 .../golang/mock/gomock/controller.go          | 235 ++++++++++
 .../github.com/golang/mock/gomock/matchers.go | 122 +++++
 7 files changed, 1136 insertions(+)
 create mode 100644 backend/vendor/github.com/golang/mock/AUTHORS
 create mode 100644 backend/vendor/github.com/golang/mock/CONTRIBUTORS
 create mode 100644 backend/vendor/github.com/golang/mock/LICENSE
 create mode 100644 backend/vendor/github.com/golang/mock/gomock/call.go
 create mode 100644 backend/vendor/github.com/golang/mock/gomock/callset.go
 create mode 100644 backend/vendor/github.com/golang/mock/gomock/controller.go
 create mode 100644 backend/vendor/github.com/golang/mock/gomock/matchers.go

diff --git a/backend/vendor/github.com/golang/mock/AUTHORS b/backend/vendor/github.com/golang/mock/AUTHORS
new file mode 100644
index 00000000..660b8ccc
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/AUTHORS
@@ -0,0 +1,12 @@
+# This is the official list of GoMock authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Alex Reece <awreece@gmail.com>
+Google Inc.
diff --git a/backend/vendor/github.com/golang/mock/CONTRIBUTORS b/backend/vendor/github.com/golang/mock/CONTRIBUTORS
new file mode 100644
index 00000000..def849ca
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/CONTRIBUTORS
@@ -0,0 +1,37 @@
+# This is the official list of people who can contribute (and typically
+# have contributed) code to the gomock repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# An entry with two email addresses specifies that the
+# first address should be used in the submit logs and
+# that the second address should be recognized as the
+# same person when interacting with Rietveld.
+
+# Please keep the list sorted.
+
+Aaron Jacobs <jacobsa@google.com> <aaronjjacobs@gmail.com>
+Alex Reece <awreece@gmail.com>
+David Symonds <dsymonds@golang.org>
+Ryan Barrett <ryanb@google.com>
diff --git a/backend/vendor/github.com/golang/mock/LICENSE b/backend/vendor/github.com/golang/mock/LICENSE
new file mode 100644
index 00000000..d6456956
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/backend/vendor/github.com/golang/mock/gomock/call.go b/backend/vendor/github.com/golang/mock/gomock/call.go
new file mode 100644
index 00000000..3d54d9f5
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/gomock/call.go
@@ -0,0 +1,420 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// Call represents an expected call to a mock.
+type Call struct {
+	t TestHelper // for triggering test failures on invalid call setup
+
+	receiver   interface{}  // the receiver of the method call
+	method     string       // the name of the method
+	methodType reflect.Type // the type of the method
+	args       []Matcher    // the args
+	origin     string       // file and line number of call setup
+
+	preReqs []*Call // prerequisite calls
+
+	// Expectations
+	minCalls, maxCalls int
+
+	numCalls int // actual number made
+
+	// actions are called when this Call is called. Each action gets the args and
+	// can set the return values by returning a non-nil slice. Actions run in the
+	// order they are created.
+	actions []func([]interface{}) []interface{}
+}
+
+// newCall creates a *Call. It requires the method type in order to support
+// unexported methods.
+func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	t.Helper()
+
+	// TODO: check arity, types.
+	margs := make([]Matcher, len(args))
+	for i, arg := range args {
+		if m, ok := arg.(Matcher); ok {
+			margs[i] = m
+		} else if arg == nil {
+			// Handle nil specially so that passing a nil interface value
+			// will match the typed nils of concrete args.
+			margs[i] = Nil()
+		} else {
+			margs[i] = Eq(arg)
+		}
+	}
+
+	origin := callerInfo(3)
+	actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
+		// Synthesize the zero value for each of the return args' types.
+		rets := make([]interface{}, methodType.NumOut())
+		for i := 0; i < methodType.NumOut(); i++ {
+			rets[i] = reflect.Zero(methodType.Out(i)).Interface()
+		}
+		return rets
+	}}
+	return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
+		args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
+}
+
+// AnyTimes allows the expectation to be called 0 or more times
+func (c *Call) AnyTimes() *Call {
+	c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity
+	return c
+}
+
+// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also
+// sets the maximum number of calls to infinity.
+func (c *Call) MinTimes(n int) *Call {
+	c.minCalls = n
+	if c.maxCalls == 1 {
+		c.maxCalls = 1e8
+	}
+	return c
+}
+
+// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also
+// sets the minimum number of calls to 0.
+func (c *Call) MaxTimes(n int) *Call {
+	c.maxCalls = n
+	if c.minCalls == 1 {
+		c.minCalls = 0
+	}
+	return c
+}
+
+// DoAndReturn declares the action to run when the call is matched.
+// The return values from this function are returned by the mocked function.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) DoAndReturn(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		vrets := v.Call(vargs)
+		rets := make([]interface{}, len(vrets))
+		for i, ret := range vrets {
+			rets[i] = ret.Interface()
+		}
+		return rets
+	})
+	return c
+}
+
+// Do declares the action to run when the call is matched. The function's
+// return values are ignored to retain backward compatibility. To use the
+// return values call DoAndReturn.
+// It takes an interface{} argument to support n-arity functions.
+func (c *Call) Do(f interface{}) *Call {
+	// TODO: Check arity and types here, rather than dying badly elsewhere.
+	v := reflect.ValueOf(f)
+
+	c.addAction(func(args []interface{}) []interface{} {
+		vargs := make([]reflect.Value, len(args))
+		ft := v.Type()
+		for i := 0; i < len(args); i++ {
+			if args[i] != nil {
+				vargs[i] = reflect.ValueOf(args[i])
+			} else {
+				// Use the zero value for the arg.
+				vargs[i] = reflect.Zero(ft.In(i))
+			}
+		}
+		v.Call(vargs)
+		return nil
+	})
+	return c
+}
+
+// Return declares the values to be returned by the mocked function call.
+func (c *Call) Return(rets ...interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	if len(rets) != mt.NumOut() {
+		c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]",
+			c.receiver, c.method, len(rets), mt.NumOut(), c.origin)
+	}
+	for i, ret := range rets {
+		if got, want := reflect.TypeOf(ret), mt.Out(i); got == want {
+			// Identical types; nothing to do.
+		} else if got == nil {
+			// Nil needs special handling.
+			switch want.Kind() {
+			case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+				// ok
+			default:
+				c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]",
+					i, c.receiver, c.method, want, c.origin)
+			}
+		} else if got.AssignableTo(want) {
+			// Assignable type relation. Make the assignment now so that the generated code
+			// can return the values with a type assertion.
+			v := reflect.New(want).Elem()
+			v.Set(reflect.ValueOf(ret))
+			rets[i] = v.Interface()
+		} else {
+			c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]",
+				i, c.receiver, c.method, got, want, c.origin)
+		}
+	}
+
+	c.addAction(func([]interface{}) []interface{} {
+		return rets
+	})
+
+	return c
+}
+
+// Times declares the exact number of times a function call is expected to be executed.
+func (c *Call) Times(n int) *Call {
+	c.minCalls, c.maxCalls = n, n
+	return c
+}
+
+// SetArg declares an action that will set the nth argument's value,
+// indirected through a pointer. Or, in the case of a slice, SetArg
+// will copy value's elements into the nth argument.
+func (c *Call) SetArg(n int, value interface{}) *Call {
+	c.t.Helper()
+
+	mt := c.methodType
+	// TODO: This will break on variadic methods.
+	// We will need to check those at invocation time.
+	if n < 0 || n >= mt.NumIn() {
+		c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]",
+			n, mt.NumIn(), c.origin)
+	}
+	// Permit setting argument through an interface.
+	// In the interface case, we don't (nay, can't) check the type here.
+	at := mt.In(n)
+	switch at.Kind() {
+	case reflect.Ptr:
+		dt := at.Elem()
+		if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) {
+			c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]",
+				n, vt, dt, c.origin)
+		}
+	case reflect.Interface:
+		// nothing to do
+	case reflect.Slice:
+		// nothing to do
+	default:
+		c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]",
+			n, at, c.origin)
+	}
+
+	c.addAction(func(args []interface{}) []interface{} {
+		v := reflect.ValueOf(value)
+		switch reflect.TypeOf(args[n]).Kind() {
+		case reflect.Slice:
+			setSlice(args[n], v)
+		default:
+			reflect.ValueOf(args[n]).Elem().Set(v)
+		}
+		return nil
+	})
+	return c
+}
+
+// isPreReq returns true if other is a direct or indirect prerequisite to c.
+func (c *Call) isPreReq(other *Call) bool {
+	for _, preReq := range c.preReqs {
+		if other == preReq || preReq.isPreReq(other) {
+			return true
+		}
+	}
+	return false
+}
+
+// After declares that the call may only match after preReq has been exhausted.
+func (c *Call) After(preReq *Call) *Call {
+	c.t.Helper()
+
+	if c == preReq {
+		c.t.Fatalf("A call isn't allowed to be its own prerequisite")
+	}
+	if preReq.isPreReq(c) {
+		c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq)
+	}
+
+	c.preReqs = append(c.preReqs, preReq)
+	return c
+}
+
+// Returns true if the minimum number of calls have been made.
+func (c *Call) satisfied() bool {
+	return c.numCalls >= c.minCalls
+}
+
+// Returns true iff the maximum number of calls have been made.
+func (c *Call) exhausted() bool {
+	return c.numCalls >= c.maxCalls
+}
+
+func (c *Call) String() string {
+	args := make([]string, len(c.args))
+	for i, arg := range c.args {
+		args[i] = arg.String()
+	}
+	arguments := strings.Join(args, ", ")
+	return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin)
+}
+
+// Tests if the given call matches the expected call.
+// If yes, returns nil. If no, returns error with message explaining why it does not match.
+func (c *Call) matches(args []interface{}) error {
+	if !c.methodType.IsVariadic() {
+		if len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+
+		for i, m := range c.args {
+			if !m.Matches(args[i]) {
+				return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+					c.origin, strconv.Itoa(i), args[i], m)
+			}
+		}
+	} else {
+		if len(c.args) < c.methodType.NumIn()-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d",
+				c.origin, len(c.args), c.methodType.NumIn()-1)
+		}
+		if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
+				c.origin, len(args), len(c.args))
+		}
+		if len(args) < len(c.args)-1 {
+			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d",
+				c.origin, len(args), len(c.args)-1)
+		}
+
+		for i, m := range c.args {
+			if i < c.methodType.NumIn()-1 {
+				// Non-variadic args
+				if !m.Matches(args[i]) {
+					return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+						c.origin, strconv.Itoa(i), args[i], m)
+				}
+				continue
+			}
+			// The last arg has a possibility of a variadic argument, so let it branch
+
+			// sample: Foo(a int, b int, c ...int)
+			if i < len(c.args) && i < len(args) {
+				if m.Matches(args[i]) {
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any())
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher)
+					// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC)
+					// Got Foo(a, b) want Foo(matcherA, matcherB)
+					// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD)
+					continue
+				}
+			}
+
+			// The number of actual args don't match the number of matchers,
+			// or the last matcher is a slice and the last arg is not.
+			// If this function still matches it is because the last matcher
+			// matches all the remaining arguments or the lack of any.
+			// Convert the remaining arguments, if any, into a slice of the
+			// expected type.
+			vargsType := c.methodType.In(c.methodType.NumIn() - 1)
+			vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
+			for _, arg := range args[i:] {
+				vargs = reflect.Append(vargs, reflect.ValueOf(arg))
+			}
+			if m.Matches(vargs.Interface()) {
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
+				// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
+				// Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher)
+				break
+			}
+			// Wrong number of matchers or not match. Fail.
+			// Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
+			// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
+			// Got Foo(a, b, c) want Foo(matcherA, matcherB)
+			return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
+				c.origin, strconv.Itoa(i), args[i:], c.args[i])
+
+		}
+	}
+
+	// Check that all prerequisite calls have been satisfied.
+	for _, preReqCall := range c.preReqs {
+		if !preReqCall.satisfied() {
+			return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
+				c.origin, preReqCall, c)
+		}
+	}
+
+	// Check that the call is not exhausted.
+	if c.exhausted() {
+		return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin)
+	}
+
+	return nil
+}
+
+// dropPrereqs tells the expected Call to not re-check prerequisite calls any
+// longer, and to return its current set.
+func (c *Call) dropPrereqs() (preReqs []*Call) {
+	preReqs = c.preReqs
+	c.preReqs = nil
+	return
+}
+
+func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} {
+	c.numCalls++
+	return c.actions
+}
+
+// InOrder declares that the given calls should occur in order.
+func InOrder(calls ...*Call) {
+	for i := 1; i < len(calls); i++ {
+		calls[i].After(calls[i-1])
+	}
+}
+
+func setSlice(arg interface{}, v reflect.Value) {
+	va := reflect.ValueOf(arg)
+	for i := 0; i < v.Len(); i++ {
+		va.Index(i).Set(v.Index(i))
+	}
+}
+
+func (c *Call) addAction(action func([]interface{}) []interface{}) {
+	c.actions = append(c.actions, action)
+}
diff --git a/backend/vendor/github.com/golang/mock/gomock/callset.go b/backend/vendor/github.com/golang/mock/gomock/callset.go
new file mode 100644
index 00000000..c44a8a58
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/gomock/callset.go
@@ -0,0 +1,108 @@
+// Copyright 2011 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// callSet represents a set of expected calls, indexed by receiver and method
+// name.
+type callSet struct {
+	// Calls that are still expected.
+	expected map[callSetKey][]*Call
+	// Calls that have been exhausted.
+	exhausted map[callSetKey][]*Call
+}
+
+// callSetKey is the key in the maps in callSet
+type callSetKey struct {
+	receiver interface{}
+	fname    string
+}
+
+func newCallSet() *callSet {
+	return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)}
+}
+
+// Add adds a new expected call.
+func (cs callSet) Add(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	m := cs.expected
+	if call.exhausted() {
+		m = cs.exhausted
+	}
+	m[key] = append(m[key], call)
+}
+
+// Remove removes an expected call.
+func (cs callSet) Remove(call *Call) {
+	key := callSetKey{call.receiver, call.method}
+	calls := cs.expected[key]
+	for i, c := range calls {
+		if c == call {
+			// maintain order for remaining calls
+			cs.expected[key] = append(calls[:i], calls[i+1:]...)
+			cs.exhausted[key] = append(cs.exhausted[key], call)
+			break
+		}
+	}
+}
+
+// FindMatch searches for a matching call. Returns error with explanation message if no call matched.
+func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) {
+	key := callSetKey{receiver, method}
+
+	// Search through the expected calls.
+	expected := cs.expected[key]
+	var callsErrors bytes.Buffer
+	for _, call := range expected {
+		err := call.matches(args)
+		if err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		} else {
+			return call, nil
+		}
+	}
+
+	// If we haven't found a match then search through the exhausted calls so we
+	// get useful error messages.
+	exhausted := cs.exhausted[key]
+	for _, call := range exhausted {
+		if err := call.matches(args); err != nil {
+			fmt.Fprintf(&callsErrors, "\n%v", err)
+		}
+	}
+
+	if len(expected)+len(exhausted) == 0 {
+		fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
+	}
+
+	return nil, fmt.Errorf(callsErrors.String())
+}
+
+// Failures returns the calls that are not satisfied.
+func (cs callSet) Failures() []*Call {
+	failures := make([]*Call, 0, len(cs.expected))
+	for _, calls := range cs.expected {
+		for _, call := range calls {
+			if !call.satisfied() {
+				failures = append(failures, call)
+			}
+		}
+	}
+	return failures
+}
diff --git a/backend/vendor/github.com/golang/mock/gomock/controller.go b/backend/vendor/github.com/golang/mock/gomock/controller.go
new file mode 100644
index 00000000..6fde25f5
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/gomock/controller.go
@@ -0,0 +1,235 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// GoMock - a mock framework for Go.
+//
+// Standard usage:
+//   (1) Define an interface that you wish to mock.
+//         type MyInterface interface {
+//           SomeMethod(x int64, y string)
+//         }
+//   (2) Use mockgen to generate a mock from the interface.
+//   (3) Use the mock in a test:
+//         func TestMyThing(t *testing.T) {
+//           mockCtrl := gomock.NewController(t)
+//           defer mockCtrl.Finish()
+//
+//           mockObj := something.NewMockMyInterface(mockCtrl)
+//           mockObj.EXPECT().SomeMethod(4, "blah")
+//           // pass mockObj to a real object and play with it.
+//         }
+//
+// By default, expected calls are not enforced to run in any particular order.
+// Call order dependency can be enforced by use of InOrder and/or Call.After.
+// Call.After can create more varied call order dependencies, but InOrder is
+// often more convenient.
+//
+// The following examples create equivalent call order dependencies.
+//
+// Example of using Call.After to chain expected call order:
+//
+//     firstCall := mockObj.EXPECT().SomeMethod(1, "first")
+//     secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall)
+//     mockObj.EXPECT().SomeMethod(3, "third").After(secondCall)
+//
+// Example of using InOrder to declare expected call order:
+//
+//     gomock.InOrder(
+//         mockObj.EXPECT().SomeMethod(1, "first"),
+//         mockObj.EXPECT().SomeMethod(2, "second"),
+//         mockObj.EXPECT().SomeMethod(3, "third"),
+//     )
+//
+// TODO:
+//	- Handle different argument/return types (e.g. ..., chan, map, interface).
+package gomock
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"runtime"
+	"sync"
+)
+
+// A TestReporter is something that can be used to report test failures.
+// It is satisfied by the standard library's *testing.T.
+type TestReporter interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+}
+
+// TestHelper is a TestReporter that has the Helper method.  It is satisfied
+// by the standard library's *testing.T.
+type TestHelper interface {
+	TestReporter
+	Helper()
+}
+
+// A Controller represents the top-level control of a mock ecosystem.
+// It defines the scope and lifetime of mock objects, as well as their expectations.
+// It is safe to call Controller's methods from multiple goroutines.
+type Controller struct {
+	// T should only be called within a generated mock. It is not intended to
+	// be used in user code and may be changed in future versions. T is the
+	// TestReporter passed in when creating the Controller via NewController.
+	// If the TestReporter does not implment a TestHelper it will be wrapped
+	// with a nopTestHelper.
+	T             TestHelper
+	mu            sync.Mutex
+	expectedCalls *callSet
+	finished      bool
+}
+
+func NewController(t TestReporter) *Controller {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	return &Controller{
+		T:             h,
+		expectedCalls: newCallSet(),
+	}
+}
+
+type cancelReporter struct {
+	TestHelper
+	cancel func()
+}
+
+func (r *cancelReporter) Errorf(format string, args ...interface{}) {
+	r.TestHelper.Errorf(format, args...)
+}
+func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
+	defer r.cancel()
+	r.TestHelper.Fatalf(format, args...)
+}
+
+// WithContext returns a new Controller and a Context, which is cancelled on any
+// fatal failure.
+func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
+	h, ok := t.(TestHelper)
+	if !ok {
+		h = nopTestHelper{t}
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	return NewController(&cancelReporter{h, cancel}), ctx
+}
+
+type nopTestHelper struct {
+	TestReporter
+}
+
+func (h nopTestHelper) Helper() {}
+
+func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	recv := reflect.ValueOf(receiver)
+	for i := 0; i < recv.Type().NumMethod(); i++ {
+		if recv.Type().Method(i).Name == method {
+			return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...)
+		}
+	}
+	ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver)
+	panic("unreachable")
+}
+
+func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
+	ctrl.T.Helper()
+
+	call := newCall(ctrl.T, receiver, method, methodType, args...)
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+	ctrl.expectedCalls.Add(call)
+
+	return call
+}
+
+func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} {
+	ctrl.T.Helper()
+
+	// Nest this code so we can use defer to make sure the lock is released.
+	actions := func() []func([]interface{}) []interface{} {
+		ctrl.T.Helper()
+		ctrl.mu.Lock()
+		defer ctrl.mu.Unlock()
+
+		expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
+		if err != nil {
+			origin := callerInfo(2)
+			ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
+		}
+
+		// Two things happen here:
+		// * the matching call no longer needs to check prerequite calls,
+		// * and the prerequite calls are no longer expected, so remove them.
+		preReqCalls := expected.dropPrereqs()
+		for _, preReqCall := range preReqCalls {
+			ctrl.expectedCalls.Remove(preReqCall)
+		}
+
+		actions := expected.call(args)
+		if expected.exhausted() {
+			ctrl.expectedCalls.Remove(expected)
+		}
+		return actions
+	}()
+
+	var rets []interface{}
+	for _, action := range actions {
+		if r := action(args); r != nil {
+			rets = r
+		}
+	}
+
+	return rets
+}
+
+func (ctrl *Controller) Finish() {
+	ctrl.T.Helper()
+
+	ctrl.mu.Lock()
+	defer ctrl.mu.Unlock()
+
+	if ctrl.finished {
+		ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
+	}
+	ctrl.finished = true
+
+	// If we're currently panicking, probably because this is a deferred call,
+	// pass through the panic.
+	if err := recover(); err != nil {
+		panic(err)
+	}
+
+	// Check that all remaining expected calls are satisfied.
+	failures := ctrl.expectedCalls.Failures()
+	for _, call := range failures {
+		ctrl.T.Errorf("missing call(s) to %v", call)
+	}
+	if len(failures) != 0 {
+		ctrl.T.Fatalf("aborting test due to missing call(s)")
+	}
+}
+
+func callerInfo(skip int) string {
+	if _, file, line, ok := runtime.Caller(skip + 1); ok {
+		return fmt.Sprintf("%s:%d", file, line)
+	}
+	return "unknown file"
+}
diff --git a/backend/vendor/github.com/golang/mock/gomock/matchers.go b/backend/vendor/github.com/golang/mock/gomock/matchers.go
new file mode 100644
index 00000000..189796f8
--- /dev/null
+++ b/backend/vendor/github.com/golang/mock/gomock/matchers.go
@@ -0,0 +1,122 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gomock
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// A Matcher is a representation of a class of values.
+// It is used to represent the valid or expected arguments to a mocked method.
+type Matcher interface {
+	// Matches returns whether x is a match.
+	Matches(x interface{}) bool
+
+	// String describes what the matcher matches.
+	String() string
+}
+
+type anyMatcher struct{}
+
+func (anyMatcher) Matches(x interface{}) bool {
+	return true
+}
+
+func (anyMatcher) String() string {
+	return "is anything"
+}
+
+type eqMatcher struct {
+	x interface{}
+}
+
+func (e eqMatcher) Matches(x interface{}) bool {
+	return reflect.DeepEqual(e.x, x)
+}
+
+func (e eqMatcher) String() string {
+	return fmt.Sprintf("is equal to %v", e.x)
+}
+
+type nilMatcher struct{}
+
+func (nilMatcher) Matches(x interface{}) bool {
+	if x == nil {
+		return true
+	}
+
+	v := reflect.ValueOf(x)
+	switch v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map,
+		reflect.Ptr, reflect.Slice:
+		return v.IsNil()
+	}
+
+	return false
+}
+
+func (nilMatcher) String() string {
+	return "is nil"
+}
+
+type notMatcher struct {
+	m Matcher
+}
+
+func (n notMatcher) Matches(x interface{}) bool {
+	return !n.m.Matches(x)
+}
+
+func (n notMatcher) String() string {
+	// TODO: Improve this if we add a NotString method to the Matcher interface.
+	return "not(" + n.m.String() + ")"
+}
+
+type assignableToTypeOfMatcher struct {
+	targetType reflect.Type
+}
+
+func (m assignableToTypeOfMatcher) Matches(x interface{}) bool {
+	return reflect.TypeOf(x).AssignableTo(m.targetType)
+}
+
+func (m assignableToTypeOfMatcher) String() string {
+	return "is assignable to " + m.targetType.Name()
+}
+
+// Constructors
+func Any() Matcher             { return anyMatcher{} }
+func Eq(x interface{}) Matcher { return eqMatcher{x} }
+func Nil() Matcher             { return nilMatcher{} }
+func Not(x interface{}) Matcher {
+	if m, ok := x.(Matcher); ok {
+		return notMatcher{m}
+	}
+	return notMatcher{Eq(x)}
+}
+
+// AssignableToTypeOf is a Matcher that matches if the parameter to the mock
+// function is assignable to the type of the parameter to this function.
+//
+// Example usage:
+//
+// 		dbMock.EXPECT().
+// 			Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})).
+// 			Return(errors.New("DB error"))
+//
+func AssignableToTypeOf(x interface{}) Matcher {
+	return assignableToTypeOfMatcher{reflect.TypeOf(x)}
+}

From 8aa24341c35f057c18b8f0d2668346ca52685053 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sat, 23 Mar 2019 23:30:53 -0500
Subject: [PATCH 19/45] make image ids extraction safe for pos-edits, delays
 comment parsing

---
 backend/app/store/image/fs_store.go   |  2 +-
 backend/app/store/image/image.go      | 23 ++++++++++++-----------
 backend/app/store/image/image_test.go | 12 ++++++------
 backend/app/store/service/service.go  | 22 +++++++++++++++++-----
 4 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go
index ae6c4c64..b65de287 100644
--- a/backend/app/store/image/fs_store.go
+++ b/backend/app/store/image/fs_store.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"hash/crc64"
 	"io"
-	"log"
 	"math"
 	"os"
 	"path"
@@ -15,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 8c5b620f..5f370014 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -8,13 +8,13 @@ package image
 import (
 	"context"
 	"io"
-	"log"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
+	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 )
 
@@ -42,17 +42,18 @@ type Service struct {
 const submitQueueSize = 5000
 
 type submitReq struct {
-	ID string
-	TS time.Time
+	idsFn func() (ids []string)
+	TS    time.Time
 }
 
-// Submit multiple ids for delayed commit
-func (s *Service) Submit(ids []string) {
-	if len(ids) == 0 {
+// Submit multiple ids via function for delayed commit
+func (s *Service) Submit(idsFn func() []string) {
+	if idsFn == nil {
 		return
 	}
 
 	s.once.Do(func() {
+		log.Printf("[DEBUG] image submiter activate")
 		s.submitCh = make(chan submitReq, submitQueueSize)
 		s.wg.Add(1)
 		go func() {
@@ -62,17 +63,17 @@ func (s *Service) Submit(ids []string) {
 				for atomic.LoadInt32(&s.term) == 0 && time.Since(req.TS) <= s.TTL {
 					time.Sleep(time.Millisecond * 10) // small sleep to relive busy wait but keep reactive for term (close)
 				}
-				if err := s.Commit(req.ID); err != nil {
-					log.Printf("[WARN] failed to commit image %s", req.ID)
+				for _, id := range req.idsFn() {
+					if err := s.Commit(id); err != nil {
+						log.Printf("[WARN] failed to commit image %s", id)
+					}
 				}
 			}
 			log.Printf("[INFO] image submiter terminated")
 		}()
 	})
 
-	for _, id := range ids {
-		s.submitCh <- submitReq{ID: id, TS: time.Now()}
-	}
+	s.submitCh <- submitReq{idsFn: idsFn, TS: time.Now()}
 }
 
 // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png
diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go
index 335327fa..f26eec8a 100644
--- a/backend/app/store/image/image_test.go
+++ b/backend/app/store/image/image_test.go
@@ -41,8 +41,8 @@ func TestService_Submit(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
-	svc.Submit([]string{"id1", "id2", "id3"})
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	time.Sleep(time.Millisecond * 500)
 }
@@ -54,8 +54,8 @@ func TestService_Close(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(5) // all 5 should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 500}
-	svc.Submit([]string{"id1", "id2", "id3"})
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 	svc.Close()
 }
@@ -70,8 +70,8 @@ func TestService_SubmitDelay(t *testing.T) {
 
 	store.EXPECT().Commit(gomock.Any()).Times(3) // first batch should be committed
 	svc := Service{Store: store, ImageAPI: "/blah/", TTL: time.Millisecond * 100}
-	svc.Submit([]string{"id1", "id2", "id3"})
+	svc.Submit(func() []string { return []string{"id1", "id2", "id3"} })
 	time.Sleep(150 * time.Millisecond) // let first batch to pass TTL
-	svc.Submit([]string{"id4", "id5"})
+	svc.Submit(func() []string { return []string{"id4", "id5"} })
 	svc.Submit(nil)
 }
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index d65f5cbc..6f9634ac 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -92,11 +92,23 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
-	imgIds, err := s.ImageService.ExtractPictures(comment.Text)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to prepare extract pictures")
-	}
-	s.ImageService.Submit(imgIds) // submit images commit, delayed by EditDuration
+	// submit comment images to delayed processing
+	s.ImageService.Submit(func() []string {
+		c := comment
+		cc, e := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if e != nil {
+			return nil
+		}
+		imgIds, e := s.ImageService.ExtractPictures(cc.Text)
+		if err != nil {
+			return nil
+		}
+		if len(imgIds) > 0 {
+			log.Printf("[DEBUG] image ids extracted from %s - %+v", c.ID, imgIds)
+		}
+		return imgIds
+	})
+
 	return s.Interface.Create(comment)
 }
 

From ded10dde6c5cda490a5de18bedaf5f9ee48a7398 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 24 Mar 2019 00:23:46 -0500
Subject: [PATCH 20/45] integration test for create comment with images

---
 backend/app/rest/api/rest_private_test.go | 74 +++++++++++++++++++++++
 backend/app/store/image/fs_store.go       |  1 +
 backend/app/store/image/image.go          |  6 +-
 backend/app/store/service/service.go      | 16 +++--
 4 files changed, 88 insertions(+), 9 deletions(-)

diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index cf053e2a..33663b6e 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -9,15 +9,18 @@ import (
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 func TestRest_Create(t *testing.T) {
@@ -531,3 +534,74 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 	assert.Equal(t, "file content 123", string(body))
 	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
 }
+
+func TestRest_CreateWithPictures(t *testing.T) {
+	ts, svc, teardown := startupT(t)
+	defer func() {
+		teardown()
+		os.RemoveAll("/tmp/remark42")
+	}()
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	svc.ImageService = &image.Service{
+		Store: &image.FileSystem{
+			Staging:  "/tmp/remark42/images.staging",
+			Location: "/tmp/remark42/images",
+			MaxSize:  1000,
+		},
+		TTL: time.Millisecond * 100,
+	}
+	svc.DataService.EditDuration = time.Millisecond * 100
+	svc.DataService.ImageService = svc.ImageService
+
+	uploadPicture := func(file, content string) (id string) {
+		r := strings.NewReader(content)
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", file)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, r)
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.Contains(t, m["id"], ".png")
+		return m["id"]
+	}
+
+	id1 := uploadPicture("pic1.png", "file content 123")
+	id2 := uploadPicture("pic2.png", "file content 12345")
+	id3 := uploadPicture("pic3.png", "file content xyz12365789")
+
+	text := fmt.Sprintf(`text 123  ![](/api/v1/picture/%s) *xxx* ![](/api/v1/picture/%s) ![](/api/v1/picture/%s)`, id1, id2, id3)
+	body := fmt.Sprintf(`{"text": "%s", "locator":{"url": "https://radio-t.com/blah1", "site": "radio-t"}}`, text)
+
+	resp, err := post(t, ts.URL+"/api/v1/comment", body)
+	assert.Nil(t, err)
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.Nil(t, err)
+	require.Equal(t, http.StatusCreated, resp.StatusCode, string(b))
+
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NotNil(t, err, "not moved from staging yet")
+
+	time.Sleep(300 * time.Millisecond)
+	_, err = os.Stat("/tmp/remark42/images/" + id1)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id2)
+	assert.NoError(t, err, "moved from staging")
+	_, err = os.Stat("/tmp/remark42/images/" + id3)
+	assert.NoError(t, err, "moved from staging")
+}
diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go
index b65de287..79049635 100644
--- a/backend/app/store/image/fs_store.go
+++ b/backend/app/store/image/fs_store.go
@@ -74,6 +74,7 @@ func (f *FileSystem) Save(fileName string, userID string, r io.Reader) (id strin
 
 // Commit file stored in staging location by moving it to permanent location
 func (f *FileSystem) Commit(id string) error {
+	log.Printf("[DEBUG] commit image %s", id)
 	stagingImage, permImage := f.location(f.Staging, id), f.location(f.Location, id)
 
 	if err := os.MkdirAll(path.Dir(permImage), 0700); err != nil {
diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go
index 5f370014..69230375 100644
--- a/backend/app/store/image/image.go
+++ b/backend/app/store/image/image.go
@@ -48,12 +48,12 @@ type submitReq struct {
 
 // Submit multiple ids via function for delayed commit
 func (s *Service) Submit(idsFn func() []string) {
-	if idsFn == nil {
+	if idsFn == nil || s == nil {
 		return
 	}
 
 	s.once.Do(func() {
-		log.Printf("[DEBUG] image submiter activate")
+		log.Printf("[DEBUG] image submitter activated")
 		s.submitCh = make(chan submitReq, submitQueueSize)
 		s.wg.Add(1)
 		go func() {
@@ -69,7 +69,7 @@ func (s *Service) Submit(idsFn func() []string) {
 					}
 				}
 			}
-			log.Printf("[INFO] image submiter terminated")
+			log.Printf("[INFO] image submitter terminated")
 		}()
 	})
 
diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 6f9634ac..1a71846b 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -92,14 +92,20 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		comment.PostTitle = title
 	}()
 
-	// submit comment images to delayed processing
+	s.submitImages(comment)
+	return s.Interface.Create(comment)
+}
+
+// submitImages initiated delayed commit of all images from the comment uploaded to remark42
+func (s *DataStore) submitImages(comment store.Comment) {
+
 	s.ImageService.Submit(func() []string {
 		c := comment
-		cc, e := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
-		if e != nil {
+		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
+		if err != nil {
 			return nil
 		}
-		imgIds, e := s.ImageService.ExtractPictures(cc.Text)
+		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
 		if err != nil {
 			return nil
 		}
@@ -108,8 +114,6 @@ func (s *DataStore) Create(comment store.Comment) (commentID string, err error)
 		}
 		return imgIds
 	})
-
-	return s.Interface.Create(comment)
 }
 
 // prepareNewComment sets new comment fields, hashing and sanitizing data

From 2c0cd1dec7f0f4b83cff23b9d0853d6491dc2eb4 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 24 Mar 2019 03:16:37 -0500
Subject: [PATCH 21/45] add test for submitImages

---
 backend/app/store/service/service.go      |  2 ++
 backend/app/store/service/service_test.go | 32 +++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go
index 1a71846b..fe30a28c 100644
--- a/backend/app/store/service/service.go
+++ b/backend/app/store/service/service.go
@@ -103,10 +103,12 @@ func (s *DataStore) submitImages(comment store.Comment) {
 		c := comment
 		cc, err := s.Get(c.Locator, c.ID) // this can be called after last edit, we have to retrieve fresh comment
 		if err != nil {
+			log.Printf("[WARN] can't get comment's %s text for image extraction, %v", c.ID, err)
 			return nil
 		}
 		imgIds, err := s.ImageService.ExtractPictures(cc.Text)
 		if err != nil {
+			log.Printf("[WARN] can't get extract pictures from %s, %v", c.ID, err)
 			return nil
 		}
 		if len(imgIds) > 0 {
diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index 2612e945..0918c210 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -13,9 +13,12 @@ import (
 	"time"
 
 	bolt "github.com/coreos/bbolt"
+	"github.com/go-pkgz/lgr"
+	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
@@ -686,6 +689,35 @@ func TestService_Find(t *testing.T) {
 	assert.InDelta(t, 0, res[1].Controversy, 0.01)
 }
 
+func TestService_submitImages(t *testing.T) {
+	defer os.Remove(testDb)
+	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
+
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	mockStore := image.NewMockStore(ctrl)
+	imgSvc := &image.Service{Store: mockStore, TTL: time.Millisecond * 50}
+
+	mockStore.EXPECT().Commit(gomock.Any()).Times(2)
+
+	// two comments for https://radio-t.com
+	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 50 * time.Millisecond,
+		AdminStore: admin.NewStaticKeyStore("secret 123"), ImageService: imgSvc}
+
+	c := store.Comment{
+		ID:        "id-22",
+		Text:      `some text <img src="/images/dev/pic1.png"/> xx <img src="/images/dev/pic2.png"/>`,
+		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+		User:      store.User{ID: "user1", Name: "user name"},
+	}
+	_, err := b.Interface.Create(c) // create directly with engine, doesn't call submitImages
+	assert.NoError(t, err)
+
+	b.submitImages(c)
+	time.Sleep(250 * time.Millisecond)
+}
+
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
 	os.Remove(testDb)

From 02da07925c06e9987c8e44673150afa15e89506e Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 24 Mar 2019 16:28:12 -0500
Subject: [PATCH 22/45] merge master

---
 go.mod |   8 -----
 go.sum | 102 ---------------------------------------------------------
 2 files changed, 110 deletions(-)
 delete mode 100644 go.mod
 delete mode 100644 go.sum

diff --git a/go.mod b/go.mod
deleted file mode 100644
index a95adc32..00000000
--- a/go.mod
+++ /dev/null
@@ -1,8 +0,0 @@
-module github.com/umputun/remark
-
-go 1.12
-
-require (
-	github.com/jessevdk/go-flags v1.4.0 // indirect
-	github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e // indirect
-)
diff --git a/go.sum b/go.sum
deleted file mode 100644
index 9d5c8b96..00000000
--- a/go.sum
+++ /dev/null
@@ -1,102 +0,0 @@
-cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
-github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
-github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
-github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
-github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/didip/tollbooth v4.0.0+incompatible h1:ayQZYuF5QOxx3NdYRNuRVFLv9/2b64JtSUlewb+0TMo=
-github.com/didip/tollbooth v4.0.0+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY=
-github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d h1:vs5Nf6IE0N/PwGJ8//zRed4gpCdcr99K2HzX7RuLOQ8=
-github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d/go.mod h1:YWyIfq3y4ArRfWZ9XksmuusP+7Mad+T0iFZ0kv0XG/M=
-github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
-github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/go-chi/chi v3.3.2+incompatible h1:uQNcQN3NsV1j4ANsPh42P4ew4t6rnRbJb8frvpp31qQ=
-github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
-github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
-github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
-github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-pkgz/auth v0.4.2 h1:WY3XzjUieUGxJSjXDU0rLrKt8RcPEaGdtLVRS7M56wU=
-github.com/go-pkgz/auth v0.4.2/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
-github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
-github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
-github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
-github.com/go-pkgz/lgr v0.4.0 h1:s4490VXkaepbkMZBNZgr3rgfUg0G4nOLVa/Yp0hlwyc=
-github.com/go-pkgz/lgr v0.4.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
-github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
-github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
-github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
-github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
-github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
-github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
-github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
-github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
-github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
-github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
-github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
-github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4=
-github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8=
-github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604 h1:BbG6VMVavjbhIsD7Hoscfz+wExp1hY+pmk+7Agc4J74=
-github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
-github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
-github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rakyll/statik v0.1.3 h1:H/5HK3yNM7sDzOiMQtC2Q1N69hl+KxzomBBWus662LU=
-github.com/rakyll/statik v0.1.3/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e h1:QLjdh6YuFoVtO3yrIG9ya7zPaaIa5lSUftkRew7+Joo=
-github.com/umputun/remark/backend v0.0.0-20190310204252-034101fb648e/go.mod h1:PORk4Y+iHyF5RDqcp1axFaMMFZ+TfA88aYXMvkOz9pc=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
-golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
-golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
-golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=

From 2a64a71eb85e94d14e1a9172460aff1567c758e5 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 01:15:51 -0500
Subject: [PATCH 23/45] lint: missing err check in test

---
 backend/app/rest/api/rest_private_test.go | 2 ++
 backend/app/store/image/fs_store_test.go  | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index 33663b6e..662fe321 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -523,6 +523,7 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 
 	m := map[string]string{}
 	err = json.Unmarshal(body, &m)
+	assert.NoError(t, err)
 	assert.Contains(t, m["id"], ".png")
 
 	// load picture
@@ -577,6 +578,7 @@ func TestRest_CreateWithPictures(t *testing.T) {
 		require.Nil(t, err)
 		m := map[string]string{}
 		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
 		assert.Contains(t, m["id"], ".png")
 		return m["id"]
 	}
diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go
index 5906d814..56623131 100644
--- a/backend/app/store/image/fs_store_test.go
+++ b/backend/app/store/image/fs_store_test.go
@@ -187,14 +187,14 @@ func TestFsStore_Cleanup(t *testing.T) {
 	assert.NotNil(t, err, "no file on staging anymore")
 }
 
-func prepareImageTest(t *testing.T) (svc FileSystem, teardown func()) {
+func prepareImageTest(t *testing.T) (svc *FileSystem, teardown func()) {
 	loc, err := ioutil.TempDir("", "test_image_r42")
 	require.NoError(t, err, "failed to make temp dir")
 
 	staging, err := ioutil.TempDir("", "test_image_r42.staging")
 	require.NoError(t, err, "failed to make temp staging dir")
 
-	svc = FileSystem{
+	svc = &FileSystem{
 		Location:   loc,
 		Staging:    staging,
 		Partitions: 100,

From 4f67afffb1030129c63cc33bc8a165081a2f8e81 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 02:45:59 -0500
Subject: [PATCH 24/45] adjust partition tests

---
 backend/app/store/image/fs_store_test.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/backend/app/store/image/fs_store_test.go b/backend/app/store/image/fs_store_test.go
index 56623131..39c2abad 100644
--- a/backend/app/store/image/fs_store_test.go
+++ b/backend/app/store/image/fs_store_test.go
@@ -108,14 +108,15 @@ func TestFsStore_location(t *testing.T) {
 		id, res    string
 	}{
 		{10, "u1/abcdefg.png", "/tmp/u1/4/abcdefg.png"},
-		{10, "abcdefe", "/tmp/unknown/1/abcdefe"},
-		{10, "12345", "/tmp/unknown/9/12345"},
+		{10, "u2/abcdefe", "/tmp/u2/0/abcdefe"},
+		{10, "u3/12345", "/tmp/u3/4/12345"},
 		{100, "12345", "/tmp/unknown/69/12345"},
 		{100, "xyzz", "/tmp/unknown/58/xyzz"},
-		{100, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/02/6851dcde6024e03258a66705f29e14b506048c74.png"},
-		{5, "6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/unknown/2/6851dcde6024e03258a66705f29e14b506048c74.png"},
-		{5, "xxxyz.png", "/tmp/unknown/0/xxxyz.png"},
+		{100, "u4/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/u4/07/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "user/6851dcde6024e03258a66705f29e14b506048c74.png", "/tmp/user/1/6851dcde6024e03258a66705f29e14b506048c74.png"},
+		{5, "aa-xxxyz.png", "/tmp/unknown/3/aa-xxxyz.png"},
 		{0, "12345", "/tmp/unknown/12345"},
+		{0, "user/12345", "/tmp/user/12345"},
 	}
 	for n, tt := range tbl {
 		t.Run(strconv.Itoa(n), func(t *testing.T) {

From 7f796d5ed9bd18594567752c055bfb897f764214 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 12:02:06 -0500
Subject: [PATCH 25/45] add image related docs

---
 README.md | 126 ++++++++++++++++++++++++++++++------------------------
 1 file changed, 69 insertions(+), 57 deletions(-)

diff --git a/README.md b/README.md
index 82e3f92d..b449f9bc 100644
--- a/README.md
+++ b/README.md
@@ -86,63 +86,68 @@ _this is the recommended way to run remark42_
 
 #### Parameters
 
-| Command line            | Environment             | Default               | Description                                      |
-| ----------------------- | ----------------------- | --------------------- | ------------------------------------------------ |
-| url                     | REMARK_URL              |                       | url to remark42 server, _required_               |
-| secret                  | SECRET                  |                       | secret key, _required_                           |
-| site                    | SITE                    | `remark`              | site name(s), _multi_                            |
-| store.type              | STORE_TYPE              | `bolt`                | type of storage, `bolt` or `mongo`               |
-| store.bolt.path         | STORE_BOLT_PATH         | `./var`               | path to data directory                           |
-| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                 | boltdb access timeout                            |
-| mongo.url               | MONGO_URL               |                       | mongo url for all stores using mongodb           |
-| mongo.db                | MONGO_DB                |                       | mongo database                                   |
-| admin.shared.id         | ADMIN_SHARED_ID         |                       | admin names (list of user ids), _multi_          |
-| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}` | admin email                                      |
-| backup                  | BACKUP_PATH             | `./var/backup`        | backups location                                 |
-| max-back                | MAX_BACKUP_FILES        | `10`                  | max backup files to keep                         |
-| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                | max number of cached items, `0` - unlimited      |
-| cache.max.value         | CACHE_MAX_VALUE         | `65536`               | max size of cached value, `0` - unlimited        |
-| cache.max.size          | CACHE_MAX_SIZE          | `50000000`            | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                  | type of avatar storage, `fs`, 'bolt`, or `mongo` |
-| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`       | avatars location for `fs` store                  |
-| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`    | file name for  `bolt` store                      |
-| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)        | max image size for resizing avatars on save      |
-| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                  | jwt TTL                                          |
-| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                | cookie TTL                                       |
-| auth.google.cid         | AUTH_GOOGLE_CID         |                       | Google OAuth client ID                           |
-| auth.google.csec        | AUTH_GOOGLE_CSEC        |                       | Google OAuth client secret                       |
-| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                       | Facebook OAuth client ID                         |
-| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                       | Facebook OAuth client secret                     |
-| auth.github.cid         | AUTH_GITHUB_CID         |                       | Github OAuth client ID                           |
-| auth.github.csec        | AUTH_GITHUB_CSEC        |                       | Github OAuth client secret                       |
-| auth.yandex.cid         | AUTH_YANDEX_CID         |                       | Yandex OAuth client ID                           |
-| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                       | Yandex OAuth client secret                       |
-| auth.dev                | AUTH_DEV                | `false`               | local oauth2 server, development mode only       |
-| auth.anon               | AUTH_ANON               | `false`               | enable anonymous login                           |
-| notify.type             | NOTIFY_TYPE             | none                  | type of notification (none or telegram)          |
-| notify.queue            | NOTIFY_QUEUE            | `100`                 | size of notification queue                       |
-| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                       | telegram token                                   |
-| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                       | telegram channel                                 |
-| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                  | telegram timeout                                 |
-| ssl.type                | SSL_TYPE                | none                  | `none`-http, `static`-https, `auto`-https + le   |
-| ssl.port                | SSL_PORT                | `8443`                | port for https server                            |
-| ssl.cert                | SSL_CERT                |                       | path to cert.pem file                            |
-| ssl.key                 | SSL_KEY                 |                       | path to key.pem file                             |
-| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`          | dir where obtained le-certs will be stored       |
-| ssl.acme-email          | SSL_ACME_EMAIL          |                       | admin email for receiving notifications from LE  |
-| max-comment             | MAX_COMMENT_SIZE        | `2048`                | comment's size limit                             |
-| max-votes               | MAX_VOTES               | `-1`                  | votes limit per comment, `-1` - unlimited        |
-| low-score               | LOW_SCORE               | `-5`                  | low score threshold                              |
-| positive-score          | POSITIVE_SCORE          | `false`               | enable positive score only                       |
-| critical-score          | CRITICAL_SCORE          | `-10`                 | critical score threshold                         |
-| positive-score          | POSITIVE_SCORE          | `false`               | restricts comment's score to be only positive    |
-| restricted-words        | RESTRICTED_WORDS        |                       | words banned in comments (can use `*`), _multi_  |
-| edit-time               | EDIT_TIME               | `5m`                  | edit window                                      |
-| read-age                | READONLY_AGE            |                       | read-only age of comments, days                  |
-| img-proxy               | IMG_PROXY               | `false`               | enable http->https proxy for images              |
-| update-limit            | UPDATE_LIMIT            | `0.5`                 | updates/sec limit                                |
-| admin-passwd            | ADMIN_PASSWD            | none (disabled)       | password for `admin` basic auth                  |
-| dbg                     | DEBUG                   | `false`               | debug mode                                       |
+| Command line            | Environment             | Default                  | Description                                      |
+| ----------------------- | ----------------------- | ------------------------ | ------------------------------------------------ |
+| url                     | REMARK_URL              |                          | url to remark42 server, _required_               |
+| secret                  | SECRET                  |                          | secret key, _required_                           |
+| site                    | SITE                    | `remark`                 | site name(s), _multi_                            |
+| store.type              | STORE_TYPE              | `bolt`                   | type of storage, `bolt` or `mongo`               |
+| store.bolt.path         | STORE_BOLT_PATH         | `./var`                  | path to data directory                           |
+| store.bolt.timeout      | STORE_BOLT_TIMEOUT      | `30s`                    | boltdb access timeout                            |
+| mongo.url               | MONGO_URL               |                          | mongo url for all stores using mongodb           |
+| mongo.db                | MONGO_DB                |                          | mongo database                                   |
+| admin.shared.id         | ADMIN_SHARED_ID         |                          | admin names (list of user ids), _multi_          |
+| admin.shared.email      | ADMIN_SHARED_EMAIL      | `admin@${REMARK_URL}`    | admin email                                      |
+| backup                  | BACKUP_PATH             | `./var/backup`           | backups location                                 |
+| max-back                | MAX_BACKUP_FILES        | `10`                     | max backup files to keep                         |
+| cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
+| cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
+| cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, 'bolt`, or `mongo` |
+| avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
+| avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
+| avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |
+| image.type              | IMAGE_TYPE              | `fs`                     | type of image storage, `fs`, 'bolt`, or `mongo`  |
+| image.max-size          | IMAGE_MAX_SIZE          | `5000000`                | max size of image file                           |
+| image.fs.path           | IMAGE_FS_PATH           | `./var/pictures`         | permanent location of images                     |
+| image.fs.staging        | IMAGE_FS_STAGING        | `./var/pictures.staging` | staging location of images                       |
+| image.fs.partitions     | IMAGE_FS_PARTITIONS     | `100`                    | number of image partitions                       |
+| auth.ttl.jwt            | AUTH_TTL_JWT            | `5m`                     | jwt TTL                                          |
+| auth.ttl.cookie         | AUTH_TTL_COOKIE         | `200h`                   | cookie TTL                                       |
+| auth.google.cid         | AUTH_GOOGLE_CID         |                          | Google OAuth client ID                           |
+| auth.google.csec        | AUTH_GOOGLE_CSEC        |                          | Google OAuth client secret                       |
+| auth.facebook.cid       | AUTH_FACEBOOK_CID       |                          | Facebook OAuth client ID                         |
+| auth.facebook.csec      | AUTH_FACEBOOK_CSEC      |                          | Facebook OAuth client secret                     |
+| auth.github.cid         | AUTH_GITHUB_CID         |                          | Github OAuth client ID                           |
+| auth.github.csec        | AUTH_GITHUB_CSEC        |                          | Github OAuth client secret                       |
+| auth.yandex.cid         | AUTH_YANDEX_CID         |                          | Yandex OAuth client ID                           |
+| auth.yandex.csec        | AUTH_YANDEX_CSEC        |                          | Yandex OAuth client secret                       |
+| auth.dev                | AUTH_DEV                | `false`                  | local oauth2 server, development mode only       |
+| auth.anon               | AUTH_ANON               | `false`                  | enable anonymous login                           |
+| notify.type             | NOTIFY_TYPE             | none                     | type of notification (none or telegram)          |
+| notify.queue            | NOTIFY_QUEUE            | `100`                    | size of notification queue                       |
+| notify.telegram.token   | NOTIFY_TELEGRAM_TOKEN   |                          | telegram token                                   |
+| notify.telegram.chan    | NOTIFY_TELEGRAM_CHAN    |                          | telegram channel                                 |
+| notify.telegram.timeout | NOTIFY_TELEGRAM_TIMEOUT | `5s`                     | telegram timeout                                 |
+| ssl.type                | SSL_TYPE                | none                     | `none`-http, `static`-https, `auto`-https + le   |
+| ssl.port                | SSL_PORT                | `8443`                   | port for https server                            |
+| ssl.cert                | SSL_CERT                |                          | path to cert.pem file                            |
+| ssl.key                 | SSL_KEY                 |                          | path to key.pem file                             |
+| ssl.acme-location       | SSL_ACME_LOCATION       | `./var/acme`             | dir where obtained le-certs will be stored       |
+| ssl.acme-email          | SSL_ACME_EMAIL          |                          | admin email for receiving notifications from LE  |
+| max-comment             | MAX_COMMENT_SIZE        | `2048`                   | comment's size limit                             |
+| max-votes               | MAX_VOTES               | `-1`                     | votes limit per comment, `-1` - unlimited        |
+| low-score               | LOW_SCORE               | `-5`                     | low score threshold                              |
+| positive-score          | POSITIVE_SCORE          | `false`                  | enable positive score only                       |
+| critical-score          | CRITICAL_SCORE          | `-10`                    | critical score threshold                         |
+| positive-score          | POSITIVE_SCORE          | `false`                  | restricts comment's score to be only positive    |
+| restricted-words        | RESTRICTED_WORDS        |                          | words banned in comments (can use `*`), _multi_  |
+| edit-time               | EDIT_TIME               | `5m`                     | edit window                                      |
+| read-age                | READONLY_AGE            |                          | read-only age of comments, days                  |
+| img-proxy               | IMG_PROXY               | `false`                  | enable http->https proxy for images              |
+| update-limit            | UPDATE_LIMIT            | `0.5`                    | updates/sec limit                                |
+| admin-passwd            | ADMIN_PASSWD            | none (disabled)          | password for `admin` basic auth                  |
+| dbg                     | DEBUG                   | `false`                  | debug mode                                       |
 
 * command line parameters are long form `--<key>=value`, i.e. `--site=https://demo.remark42.com`
 * _multi_ parameters separated by `,` in the environment or repeated with command line key, like `--site=s1 --site=s2 ...`
@@ -602,6 +607,13 @@ Sort can be `time`, `active` or `score`. Supported sort order with prefix -/+, i
 * `GET /api/v1/rss/site?site=site-id` - rss feed for given site
 * `GET /api/v1/rss/reply?site=site-id&user=user-id` - rss feed for replies to user's comments
 
+### Images management
+
+* `GET /api/v1/picture/{user}/{id}` - load stored image 
+* `POST /api/v1/picture` - upload and store image, uses post form with `FormFile("file")`. returns `{"id": user/imgid}` _auth required_
+
+_returned id should be appended to load image url on caller side_
+  
 ### Admin
 
 * `DELETE /api/v1/admin/comment/{id}?site=site-id&url=post-url` - delete comment by `id`.

From 542a1e99571e8d076186a06aa3dd8cda051aa9de Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 18:05:14 -0500
Subject: [PATCH 26/45] missing vendor flag in build cmd

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 05e7eff2..8fec5895 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,7 +54,7 @@ RUN \
     echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
-    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
+    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 
 FROM node:10.11-alpine as build-frontend-deps

From 7a948266a376187cfd0a724ff6d656c63ed9f39b Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 19:54:16 -0500
Subject: [PATCH 27/45] test more image types

---
 backend/app/rest/api/rest_private_test.go | 79 +++++++++++++++--------
 1 file changed, 52 insertions(+), 27 deletions(-)

diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index 662fe321..e43d7ff1 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -500,40 +500,65 @@ func TestRest_SavePictureCtrl(t *testing.T) {
 	defer teardown()
 
 	// save picture
-	r := strings.NewReader("file content 123")
-	bodyBuf := &bytes.Buffer{}
-	bodyWriter := multipart.NewWriter(bodyBuf)
-	fileWriter, err := bodyWriter.CreateFormFile("file", "picture.png")
-	require.NoError(t, err)
-	_, err = io.Copy(fileWriter, r)
-	require.NoError(t, err)
-	contentType := bodyWriter.FormDataContentType()
-	require.NoError(t, bodyWriter.Close())
+	savePic := func(name string) (id string) {
+		r := strings.NewReader("file content 123")
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("file", name)
+		require.NoError(t, err)
+		_, err = io.Copy(fileWriter, r)
+		require.NoError(t, err)
+		contentType := bodyWriter.FormDataContentType()
+		require.NoError(t, bodyWriter.Close())
 
-	client := http.Client{}
-	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		client := http.Client{}
+		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/picture", ts.URL), bodyBuf)
+		require.NoError(t, err)
+		req.Header.Add("Content-Type", contentType)
+		req.Header.Add("X-JWT", devToken)
+		resp, err := client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		body, err := ioutil.ReadAll(resp.Body)
+		require.Nil(t, err)
+
+		m := map[string]string{}
+		err = json.Unmarshal(body, &m)
+		assert.NoError(t, err)
+		assert.True(t, m["id"] != "")
+		return m["id"]
+	}
+
+	id := savePic("picture.png")
+	resp, err := http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
 	require.NoError(t, err)
-	req.Header.Add("Content-Type", contentType)
-	req.Header.Add("X-JWT", devToken)
-	resp, err := client.Do(req)
-	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
 	require.Nil(t, err)
-
-	m := map[string]string{}
-	err = json.Unmarshal(body, &m)
-	assert.NoError(t, err)
-	assert.Contains(t, m["id"], ".png")
-
-	// load picture
-	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, m["id"]))
-	require.NoError(t, err)
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err = ioutil.ReadAll(resp.Body)
-	require.Nil(t, err)
 	assert.Equal(t, "file content 123", string(body))
 	assert.Equal(t, "image/png", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.gif")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/gif", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.jpg")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/jpeg", resp.Header.Get("Content-Type"))
+
+	id = savePic("picture.blah")
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/%s", ts.URL, id))
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+	assert.Equal(t, "image/*", resp.Header.Get("Content-Type"))
+
+	resp, err = http.Get(fmt.Sprintf("%s/api/v1/picture/blah/pic.blah", ts.URL))
+	require.NoError(t, err)
+	assert.Equal(t, 400, resp.StatusCode)
 }
 
 func TestRest_CreateWithPictures(t *testing.T) {

From 38da69ea49b6f5da30a9c54d04138d03f21bdcdb Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 20:58:57 -0500
Subject: [PATCH 28/45] simplify counts hashing key

---
 backend/app/rest/api/rest_public.go      | 11 +++------
 backend/app/rest/api/rest_public_test.go | 31 ++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/backend/app/rest/api/rest_public.go b/backend/app/rest/api/rest_public.go
index 3713ef53..3657bacd 100644
--- a/backend/app/rest/api/rest_public.go
+++ b/backend/app/rest/api/rest_public.go
@@ -1,7 +1,7 @@
 package api
 
 import (
-	"crypto/sha1" //nolint
+	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
 	"net/http"
@@ -275,12 +275,9 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	hasher := sha1.New() //nolint
-	if _, err := hasher.Write([]byte(k)); err != nil {
-		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't make sha1 for list of urls", rest.ErrInternal)
-		return
-	}
-	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	h := sha1.Sum([]byte(k)) //nolint
+	sha := base64.URLEncoding.EncodeToString(h[:])
+
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
 	data, err := s.Cache.Get(key, func() ([]byte, error) {
 		counts, e := s.DataService.Counts(siteID, posts)
diff --git a/backend/app/rest/api/rest_public_test.go b/backend/app/rest/api/rest_public_test.go
index 788f34e2..e3343a8f 100644
--- a/backend/app/rest/api/rest_public_test.go
+++ b/backend/app/rest/api/rest_public_test.go
@@ -377,6 +377,37 @@ func TestRest_List(t *testing.T) {
 	assert.Equal(t, 3, pi[1].Count)
 }
 
+func TestRest_ListWithSkipAndLimit(t *testing.T) {
+	ts, _, teardown := startupT(t)
+	defer teardown()
+
+	c1 := store.Comment{Text: "test test #1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah1"}}
+	c2 := store.Comment{Text: "test test #2", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah2"}}
+	c3 := store.Comment{Text: "test test #3", ParentID: "p1",
+		Locator: store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah3"}}
+
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c1, ts)
+	addComment(t, c2, ts)
+	addComment(t, c2, ts)
+	addComment(t, c3, ts)
+	addComment(t, c3, ts)
+
+	body, code := get(t, ts.URL+"/api/v1/list?site=radio-t&skip=1&limit=2")
+	assert.Equal(t, 200, code)
+	pi := []store.PostInfo{}
+	err := json.Unmarshal([]byte(body), &pi)
+	assert.Nil(t, err)
+	require.Equal(t, 2, len(pi))
+	assert.Equal(t, "https://radio-t.com/blah2", pi[0].URL)
+	assert.Equal(t, 2, pi[0].Count)
+	assert.Equal(t, "https://radio-t.com/blah1", pi[1].URL)
+	assert.Equal(t, 3, pi[1].Count)
+}
+
 func TestRest_Config(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()

From cf09ea27eb7c523e685f4ef3e82fae914aedc556 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 21:10:49 -0500
Subject: [PATCH 29/45] remove mock from test coverage

---
 Dockerfile | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 05e7eff2..596312d9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,7 +29,9 @@ WORKDIR /build/backend
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
-    if [ -z "$SKIP_BACKEND_TEST" ] ; then go test -mod=vendor -covermode=count -coverprofile=/profile.cov ./... ; \
+    if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
 RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
@@ -37,10 +39,10 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
-    golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
-    --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
-    --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
-    --enable=ineffassign --enable=varcheck ./... ; \
+        golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
+        --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
+        --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \
+        --enable=ineffassign --enable=varcheck ./... ; \
     else echo "skip backend linters" ; fi
 
 # submit coverage to coverals if COVERALLS_TOKEN in env
@@ -50,8 +52,7 @@ RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
 
 # if DRONE presented use DRONE_* git env to make version
 RUN \
-    if [ -z "$DRONE" ] ; then \
-    echo "runs outside of drone" && version="local"; \
+    if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
     go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app

From 4f73ddc40c3aa52953cf95e97e6c8c4a1161f118 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 23:31:02 -0500
Subject: [PATCH 30/45] move error parsing for rest to separate func

---
 backend/app/rest/api/rest_private.go      | 46 ++++++++++++++---------
 backend/app/rest/api/rest_private_test.go | 26 +++++++++++++
 2 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go
index b7063656..f03f335e 100644
--- a/backend/app/rest/api/rest_private.go
+++ b/backend/app/rest/api/rest_private.go
@@ -130,14 +130,9 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
+
 	if err != nil {
-		code := rest.ErrCommentRejected
-		switch {
-		case strings.HasPrefix(err.Error(), "too late to edit"):
-			code = rest.ErrCommentEditExpired
-		case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
-			code = rest.ErrCommentEditChanged
-		}
+		code := s.parseError(err, rest.ErrCommentRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't update comment", code)
 		return
 	}
@@ -178,17 +173,7 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 
 	comment, err := s.DataService.Vote(locator, id, user.ID, vote)
 	if err != nil {
-		code := rest.ErrVoteRejected
-		switch {
-		case strings.Contains(err.Error(), "can not vote for his own comment"):
-			code = rest.ErrVoteSelf
-		case strings.Contains(err.Error(), "already voted for"):
-			code = rest.ErrVoteDbl
-		case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
-			code = rest.ErrVoteMax
-		case strings.Contains(err.Error(), "minimal score reached for comment"):
-			code = rest.ErrVoteMinScore
-		}
+		code := s.parseError(err, rest.ErrVoteRejected)
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment", code)
 		return
 	}
@@ -319,3 +304,28 @@ func (s *Rest) isReadOnly(locator store.Locator) bool {
 	}
 	return s.DataService.IsReadOnly(locator) // ro manually
 }
+
+func (s *Rest) parseError(err error, defaultCode int) (code int) {
+	code = defaultCode
+
+	switch {
+	// voting errors
+	case strings.Contains(err.Error(), "can not vote for his own comment"):
+		code = rest.ErrVoteSelf
+	case strings.Contains(err.Error(), "already voted for"):
+		code = rest.ErrVoteDbl
+	case strings.Contains(err.Error(), "maximum number of votes exceeded for comment"):
+		code = rest.ErrVoteMax
+	case strings.Contains(err.Error(), "minimal score reached for comment"):
+		code = rest.ErrVoteMinScore
+
+	// edit errors
+	case strings.HasPrefix(err.Error(), "too late to edit"):
+		code = rest.ErrCommentEditExpired
+	case strings.HasPrefix(err.Error(), "parent comment with reply can't be edited"):
+		code = rest.ErrCommentEditChanged
+
+	}
+
+	return code
+}
diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go
index e43d7ff1..9d9b1a13 100644
--- a/backend/app/rest/api/rest_private_test.go
+++ b/backend/app/rest/api/rest_private_test.go
@@ -10,14 +10,17 @@ import (
 	"mime/multipart"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/umputun/remark/backend/app/rest"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/image"
@@ -632,3 +635,26 @@ func TestRest_CreateWithPictures(t *testing.T) {
 	_, err = os.Stat("/tmp/remark42/images/" + id3)
 	assert.NoError(t, err, "moved from staging")
 }
+
+func TestRest_parseError(t *testing.T) {
+	tbl := []struct {
+		err error
+		res int
+	}{
+		{errors.New("can not vote for his own comment"), rest.ErrVoteSelf},
+		{errors.New("already voted for"), rest.ErrVoteDbl},
+		{errors.New("maximum number of votes exceeded for comment"), rest.ErrVoteMax},
+		{errors.New("minimal score reached for comment"), rest.ErrVoteMinScore},
+		{errors.New("too late to edit"), rest.ErrCommentEditExpired},
+		{errors.New("parent comment with reply can't be edited"), rest.ErrCommentEditChanged},
+		{errors.New("blah blah"), rest.ErrInternal},
+	}
+
+	svc := Rest{}
+	for n, tt := range tbl {
+		t.Run(strconv.Itoa(n), func(t *testing.T) {
+			res := svc.parseError(tt.err, rest.ErrInternal)
+			assert.Equal(t, tt.res, res)
+		})
+	}
+}

From 6ef88bf375ab5c6e2d90c5eed59214587dee89c5 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Mon, 25 Mar 2019 23:57:27 -0500
Subject: [PATCH 31/45] lint: tests warning

---
 backend/app/migrator/backup_test.go       | 7 ++++---
 backend/app/rest/api/admin_test.go        | 6 ++++--
 backend/app/rest/proxy/image_test.go      | 3 ++-
 backend/app/store/service/service_test.go | 4 ++--
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/backend/app/migrator/backup_test.go b/backend/app/migrator/backup_test.go
index 781e95f4..b7d95d08 100644
--- a/backend/app/migrator/backup_test.go
+++ b/backend/app/migrator/backup_test.go
@@ -16,7 +16,8 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
 
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
+
 	for i := 1; i <= 10; i++ {
 		fname := fmt.Sprintf("%s/backup-site1-201712%02d.gz", loc, i)
 		err := ioutil.WriteFile(fname, []byte("blah"), 0600)
@@ -40,7 +41,7 @@ func TestBackup_RemoveOldBackupFiles(t *testing.T) {
 func TestBackup_MakeBackup(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	bk := AutoBackup{BackupLocation: loc, SiteID: "site1", KeepMax: 3, Exporter: &mockExporter{}}
 	fname, err := bk.makeBackup()
@@ -56,7 +57,7 @@ func TestBackup_MakeBackup(t *testing.T) {
 func TestBackup_Do(t *testing.T) {
 	loc := "/tmp/remark-backups.test"
 	defer os.RemoveAll(loc)
-	os.MkdirAll(loc, 0700)
+	assert.NoError(t, os.MkdirAll(loc, 0700))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
diff --git a/backend/app/rest/api/admin_test.go b/backend/app/rest/api/admin_test.go
index d4848ab8..d3021b2d 100644
--- a/backend/app/rest/api/admin_test.go
+++ b/backend/app/rest/api/admin_test.go
@@ -113,11 +113,13 @@ func TestAdmin_Title(t *testing.T) {
 	srv.DataService.TitleExtractor = service.NewTitleExtractor(http.Client{Timeout: time.Second})
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
diff --git a/backend/app/rest/proxy/image_test.go b/backend/app/rest/proxy/image_test.go
index dc8eae0c..13d3a1b8 100644
--- a/backend/app/rest/proxy/image_test.go
+++ b/backend/app/rest/proxy/image_test.go
@@ -132,7 +132,8 @@ func imgHTTPServer(t *testing.T) *httptest.Server {
 			t.Log("http img request", r.URL)
 			w.Header().Add("Content-Length", "123")
 			w.Header().Add("Content-Type", "image/png")
-			w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			_, err := w.Write([]byte(fmt.Sprintf("%123s", "X")))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.Path == "/image/img-slow.png" {
diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index 0918c210..4012be13 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -261,7 +261,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", true)
 		}()
 	}
 	wg.Wait()
@@ -280,7 +280,7 @@ func TestService_VoteAggressive(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			val := rand.Intn(2) > 0
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, "user1", val)
 		}()
 	}
 	wg.Wait()

From 56c5cd64bc86162d7b7c8df81784a8be4b9ad499 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Tue, 26 Mar 2019 00:04:59 -0500
Subject: [PATCH 32/45] lint: more test warnings

---
 backend/app/store/service/service_test.go | 63 +++++++++++++----------
 backend/app/store/service/title_test.go   | 24 +++++----
 2 files changed, 48 insertions(+), 39 deletions(-)

diff --git a/backend/app/store/service/service_test.go b/backend/app/store/service/service_test.go
index 4012be13..73b66ee5 100644
--- a/backend/app/store/service/service_test.go
+++ b/backend/app/store/service/service_test.go
@@ -28,7 +28,7 @@ import (
 var testDb = "/tmp/test-remark.db"
 
 func TestService_CreateFromEmpty(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -52,7 +52,7 @@ func TestService_CreateFromEmpty(t *testing.T) {
 }
 
 func TestService_CreateFromPartial(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks}
 	comment := store.Comment{
@@ -79,7 +79,7 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	ks := admin.NewStaticKeyStore("secret 123")
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: ks,
 		TitleExtractor: NewTitleExtractor(http.Client{Timeout: 5 * time.Second})}
@@ -109,7 +109,7 @@ func TestService_CreateFromPartialWithTitle(t *testing.T) {
 }
 
 func TestService_SetTitle(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	var titleEnable int32
 	tss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -117,11 +117,13 @@ func TestService_SetTitle(t *testing.T) {
 			w.WriteHeader(404)
 		}
 		if r.URL.String() == "/post1" {
-			w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post1 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		if r.URL.String() == "/post2" {
-			w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>post2 blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -161,7 +163,7 @@ func TestService_SetTitle(t *testing.T) {
 }
 
 func TestService_Vote(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -207,7 +209,7 @@ func TestService_Vote(t *testing.T) {
 }
 
 func TestService_VoteLimit(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 2}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -225,7 +227,7 @@ func TestService_VoteLimit(t *testing.T) {
 }
 
 func TestService_VotesDisabled(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: 0}
 
 	_, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-1", "user2", true)
@@ -233,7 +235,7 @@ func TestService_VotesDisabled(t *testing.T) {
 }
 
 func TestService_VoteAggressive(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -293,7 +295,7 @@ func TestService_VoteAggressive(t *testing.T) {
 
 func TestService_VoteConcurrent(t *testing.T) {
 
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	comment := store.Comment{
@@ -310,10 +312,11 @@ func TestService_VoteConcurrent(t *testing.T) {
 	var wg sync.WaitGroup
 	for i := 0; i < 100; i++ {
 		wg.Add(1)
-		i := i
+		ii := i
 		go func() {
 			defer wg.Done()
-			b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID, fmt.Sprintf("user1-%d", i), true)
+			_, _ = b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[0].ID,
+				fmt.Sprintf("user1-%d", ii), true)
 		}()
 	}
 	wg.Wait()
@@ -325,7 +328,7 @@ func TestService_VoteConcurrent(t *testing.T) {
 }
 
 func TestService_VotePositive(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"),
 		MaxVotes: -1, PositiveScore: true}
 
@@ -344,7 +347,7 @@ func TestService_VotePositive(t *testing.T) {
 }
 
 func TestService_VoteControversy(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123"), MaxVotes: -1}
 
 	c, err := b.Vote(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "id-2", "user2", false)
@@ -394,7 +397,7 @@ func TestService_Controversy(t *testing.T) {
 }
 
 func TestService_Pin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -418,7 +421,7 @@ func TestService_Pin(t *testing.T) {
 }
 
 func TestService_EditComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -445,7 +448,7 @@ func TestService_EditComment(t *testing.T) {
 }
 
 func TestService_DeleteComment(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -464,7 +467,7 @@ func TestService_DeleteComment(t *testing.T) {
 }
 
 func TestService_EditCommentDurationFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond, AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -481,7 +484,7 @@ func TestService_EditCommentDurationFailed(t *testing.T) {
 }
 
 func TestService_EditCommentReplyFailed(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := DataStore{Interface: prepStoreEngine(t), AdminStore: admin.NewStaticKeyStore("secret 123")}
 
 	res, err := b.Last("radio-t", 0)
@@ -532,7 +535,7 @@ func TestService_ValidateComment(t *testing.T) {
 }
 
 func TestService_Counts(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	b := prepStoreEngine(t) // two comments for https://radio-t.com
 
 	// add one more for https://radio-t.com/2
@@ -561,7 +564,7 @@ func TestService_Counts(t *testing.T) {
 }
 
 func TestService_GetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -592,7 +595,7 @@ func TestService_GetMetas(t *testing.T) {
 }
 
 func TestService_SetMetas(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticKeyStore("secret 123")}
@@ -616,7 +619,7 @@ func TestService_SetMetas(t *testing.T) {
 }
 
 func TestService_IsAdmin(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	// two comments for https://radio-t.com
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
 		AdminStore: admin.NewStaticStore("secret 123", []string{"user2"}, "user@email.com")}
@@ -626,7 +629,7 @@ func TestService_IsAdmin(t *testing.T) {
 }
 
 func TestService_HasReplies(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
@@ -656,7 +659,7 @@ func TestService_HasReplies(t *testing.T) {
 }
 
 func TestService_Find(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 
 	// two comments for https://radio-t.com, no reply
 	b := DataStore{Interface: prepStoreEngine(t), EditDuration: 100 * time.Millisecond,
@@ -690,7 +693,7 @@ func TestService_Find(t *testing.T) {
 }
 
 func TestService_submitImages(t *testing.T) {
-	defer os.Remove(testDb)
+	defer teardown(t)
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
 	ctrl := gomock.NewController(t)
@@ -720,7 +723,7 @@ func TestService_submitImages(t *testing.T) {
 
 // makes new boltdb, put two records
 func prepStoreEngine(t *testing.T) engine.Interface {
-	os.Remove(testDb)
+	_ = os.Remove(testDb)
 
 	boltStore, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: "/tmp/test-remark.db", SiteID: "radio-t"})
 	assert.Nil(t, err)
@@ -748,3 +751,7 @@ func prepStoreEngine(t *testing.T) engine.Interface {
 
 	return b
 }
+
+func teardown(_ *testing.T) {
+	_ = os.Remove(testDb)
+}
diff --git a/backend/app/store/service/title_test.go b/backend/app/store/service/title_test.go
index 31b1eef7..aadd4072 100644
--- a/backend/app/store/service/title_test.go
+++ b/backend/app/store/service/title_test.go
@@ -44,7 +44,8 @@ func TestTitle_Get(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.String() == "/good" {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			_, err := w.Write([]byte("<html><title>blah 123</title><body> 2222</body></html>"))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -58,9 +59,9 @@ func TestTitle_Get(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/good")
-		require.Nil(t, err)
-		assert.Equal(t, "blah 123", title)
+		r, e := ex.Get(ts.URL + "/good")
+		require.Nil(t, e)
+		assert.Equal(t, "blah 123", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits))
 }
@@ -75,7 +76,8 @@ func TestTitle_GetConcurrent(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if strings.HasPrefix(r.URL.String(), "/good") {
 			atomic.AddInt32(&hits, 1)
-			w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			_, err := w.Write([]byte(fmt.Sprintf("<html><title>blah 123 %s</title><body>%s</body></html>", r.URL.String(), body)))
+			assert.NoError(t, err)
 			return
 		}
 		w.WriteHeader(404)
@@ -84,11 +86,11 @@ func TestTitle_GetConcurrent(t *testing.T) {
 	g := syncs.NewSizedGroup(10)
 
 	for i := 0; i < 100; i++ {
-		i := i
+		ii := i
 		g.Go(func(_ context.Context) {
-			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(i))
+			title, err := ex.Get(ts.URL + "/good/" + strconv.Itoa(ii))
 			require.Nil(t, err)
-			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(i), title)
+			assert.Equal(t, "blah 123 "+"/good/"+strconv.Itoa(ii), title)
 		})
 	}
 	g.Wait()
@@ -107,9 +109,9 @@ func TestTitle_GetFailed(t *testing.T) {
 	require.NotNil(t, err)
 
 	for i := 0; i < 100; i++ {
-		title, err := ex.Get(ts.URL + "/bad")
-		require.Nil(t, err)
-		assert.Equal(t, "", title)
+		r, e := ex.Get(ts.URL + "/bad")
+		require.Nil(t, e)
+		assert.Equal(t, "", r)
 	}
 	assert.Equal(t, int32(1), atomic.LoadInt32(&hits), "hit once, errors cached")
 }

From 29dc368ea8f67bd891f5fb215ec5bbcd01fa6eef Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 31 Mar 2019 13:45:44 -0500
Subject: [PATCH 33/45] revendor with fresh lgr, add logger to http server

---
 backend/app/main.go                           |   4 +-
 backend/app/rest/api/rest.go                  |   8 +
 backend/go.mod                                |   2 +-
 backend/go.sum                                |   6 +-
 .../vendor/github.com/go-pkgz/lgr/README.md   |  39 ++-
 .../vendor/github.com/go-pkgz/lgr/adaptor.go  |  30 ++
 .../github.com/go-pkgz/lgr/interface.go       |   9 +-
 .../vendor/github.com/go-pkgz/lgr/logger.go   | 314 ++++++++++++------
 backend/vendor/modules.txt                    |   2 +-
 9 files changed, 299 insertions(+), 115 deletions(-)
 create mode 100644 backend/vendor/github.com/go-pkgz/lgr/adaptor.go

diff --git a/backend/app/main.go b/backend/app/main.go
index de745613..b23bf9d3 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -62,10 +62,10 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces, log.CallerIgnore("logger"))
+		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
 		return
 	}
-	log.Setup(log.Msec, log.LevelBraces, log.CallerPkg, log.CallerIgnore("logger", "rest"))
+	log.Setup(log.Msec, log.LevelBraces, log.CallerPkg)
 }
 
 // getDump reads runtime stack and returns as a string
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index 7844217e..f02b7361 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -82,6 +82,7 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpServer = s.makeHTTPServer(port, s.routes())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		err := s.httpServer.ListenAndServe()
@@ -91,7 +92,10 @@ func (s *Rest) Run(port int) {
 
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSServer(s.SSLConfig.Port, s.routes())
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpToHTTPSRouter())
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
 		s.lock.Unlock()
 
 		go func() {
@@ -108,7 +112,11 @@ func (s *Rest) Run(port int) {
 		m := s.makeAutocertManager()
 		s.lock.Lock()
 		s.httpsServer = s.makeHTTPSAutocertServer(s.SSLConfig.Port, s.routes(), m)
+		s.httpsServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.httpServer = s.makeHTTPServer(port, s.httpChallengeRouter(m))
+		s.httpServer.ErrorLog = log.ToStdLogger(log.Default(), "WARN")
+
 		s.lock.Unlock()
 
 		go func() {
diff --git a/backend/go.mod b/backend/go.mod
index 071bec33..49c76116 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/go-chi/render v1.0.0
 	github.com/go-pkgz/auth v0.5.0
 	github.com/go-pkgz/lcw v0.2.0
-	github.com/go-pkgz/lgr v0.4.0
+	github.com/go-pkgz/lgr v0.6.0
 	github.com/go-pkgz/mongo v1.1.2
 	github.com/go-pkgz/repeater v1.1.1
 	github.com/go-pkgz/rest v1.4.0
diff --git a/backend/go.sum b/backend/go.sum
index 7327b7e7..d5d43b52 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -26,15 +26,13 @@ github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
 github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
 github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-pkgz/auth v0.4.2 h1:WY3XzjUieUGxJSjXDU0rLrKt8RcPEaGdtLVRS7M56wU=
-github.com/go-pkgz/auth v0.4.2/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
 github.com/go-pkgz/auth v0.5.0 h1:+wqppq35x83PchZNZ7SHHYLI/e8WeETFouujDLsklac=
 github.com/go-pkgz/auth v0.5.0/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
 github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
 github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
 github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
-github.com/go-pkgz/lgr v0.4.0 h1:s4490VXkaepbkMZBNZgr3rgfUg0G4nOLVa/Yp0hlwyc=
-github.com/go-pkgz/lgr v0.4.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.0 h1:Z9FRhfSyuASiF05iXRj+clTAAl8+1EfLR7SeyLIaGKg=
+github.com/go-pkgz/lgr v0.6.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
 github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
 github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
diff --git a/backend/vendor/github.com/go-pkgz/lgr/README.md b/backend/vendor/github.com/go-pkgz/lgr/README.md
index 4ac2c61e..de7aae27 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/README.md
+++ b/backend/vendor/github.com/go-pkgz/lgr/README.md
@@ -33,22 +33,53 @@ _Without `lgr.Caller*` it will drop `{caller}` part_
 `lgr.New` call accepts functional options:
 
 - `lgr.Debug` - turn debug mode on to allow messages with "DEBUG" level (filtered overwise)
+- `lgr.Out(io.Writer)` - sets the output writer, default `os.Stdout`
+- `lgr.Err(io.Writer)` - sets the error writer, default `os.Stderr`
 - `lgr.CallerFile` - adds the caller file info
 - `lgr.CallerFunc` - adds the caller function info
 - `lgr.CallerPkg` - adds the caller package
 - `lgr.LevelBraces` - wraps levels with "[" and "]"
 - `lgr.Msec` - adds milliseconds to timestamp
-- `lgr.Out(io.Writer)` - sets the output writer, default `os.Stdout`
-- `lgr.Err(io.Writer)` - sets the error writer, default `os.Stderr`
+- `lgr.Format` - sets custom template, overwrite all other formatting modifiers.
 
+#### formatting templates:
+
+Several predefined templates provided and can be passed directly to `lgr.Format`, i.e. `lgr.Format(lgr.WithMsec)`
+
+```
+	Short      = `{{.DT.Format "2006/01/02 15:04:05"}} {{.Level}} {{.Message}}`
+	WithMsec   = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} {{.Message}}`
+	WithPkg    = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerPkg}}) {{.Message}}`
+	ShortDebug = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFile}}:{{.CallerLine}}) {{.Message}}`
+	FuncDebug  = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFunc}}) {{.Message}}`
+	FullDebug  = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFile}}:{{.CallerLine}} {{.CallerFunc}}) {{.Message}}`
+```
+
+User can make a custom template and pass it directly to `lgr.Format`. For example:
+
+```go    
+    lgr.Format(`{{.Level}} - {{.DT.Format "2006-01-02T15:04:05Z07:00") - {{.CallerPkg}} - {{.Message}}`)
+```
+)
+    
 ### levels
 
-`lgr.Logf` recognizes prefixes like "INFO" or "[INFO]" as levels. The full list of supported levels - "DEBUG", "INFO", "WARN", "ERROR", "PANIC" and "FATAL"
+`lgr.Logf` recognizes prefixes like "INFO" or "[INFO]" as levels. The full list of supported levels - "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "PANIC" and "FATAL"
 
-- `DEBUG` will be filtered unless `lgr.Debug` option defined
+- `TRACE` will be filtered unless `lgr.Trace` option defined
+- `DEBUG` will be filtered unless `lgr.Debug` or `lgr.Trace` options defined
 - `INFO` and `WARN` don't have any special behavior attached
 - `ERROR` sends messages to both out and err writers
 - `PANIC` and `FATAL` send messages to both out and err writers. In addition sends dump of callers and runtime info to err only, and calls `os.Exit(1)`.
+
+### adaptors
+
+`lgr` logger can be converted to `io.Writer` or `*log.Logger`
+
+- `lgr.ToWriter(l lgr.L, level string) io.Writer` - makes io.Writer forwarding write ops to underlying `lgr.L`
+- `lgr.ToStdLogger(l lgr.L, level string) *log.Logger` - makes standard logger on top of `lgr.L`
+
+_`level` parameter is optional, if defined will enforce the level._    
   
 ### global logger
 
diff --git a/backend/vendor/github.com/go-pkgz/lgr/adaptor.go b/backend/vendor/github.com/go-pkgz/lgr/adaptor.go
new file mode 100644
index 00000000..3c213c2f
--- /dev/null
+++ b/backend/vendor/github.com/go-pkgz/lgr/adaptor.go
@@ -0,0 +1,30 @@
+package lgr
+
+import (
+	"log"
+	"strings"
+)
+
+// Writer holds lgr.L and wraps with io.Writer interface
+type Writer struct {
+	L
+	level string // if defined added to each message
+}
+
+// Write to lgr.L, trim EOL
+func (w *Writer) Write(p []byte) (n int, err error) {
+	w.Logf(strings.TrimSuffix(w.level+string(p), "\n"))
+	return len(p), nil
+}
+
+// ToWriter makes io.Writer for given lgr.L with optional level
+func ToWriter(l L, level string) *Writer {
+	if level != "" && !strings.HasSuffix(level, " ") {
+		level += " "
+	}
+	return &Writer{l, level}
+}
+
+func ToStdLogger(l L, level string) *log.Logger {
+	return log.New(ToWriter(l, level), "", 0)
+}
diff --git a/backend/vendor/github.com/go-pkgz/lgr/interface.go b/backend/vendor/github.com/go-pkgz/lgr/interface.go
index 80259cb3..dbe1969a 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/interface.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/interface.go
@@ -15,7 +15,7 @@ type L interface {
 // Func type is an adapter to allow the use of ordinary functions as Logger.
 type Func func(format string, args ...interface{})
 
-// Logf calls f(id)
+// Logf calls f(format, args...)
 func (f Func) Logf(format string, args ...interface{}) { f(format, args...) }
 
 // NoOp logger
@@ -26,24 +26,23 @@ var Std = Func(func(format string, args ...interface{}) { stdlog.Printf(format,
 
 // Printf simplifies replacement of std logger
 func Printf(format string, args ...interface{}) {
-	def.Logf(format, args...)
+	def.logf(format, args...)
 }
 
 // Print simplifies replacement of std logger
 func Print(line string) {
-	def.Logf(line)
+	def.logf(line)
 }
 
 // Fatalf simplifies replacement of std logger
 func Fatalf(format string, args ...interface{}) {
-	def.Logf(format, args...)
+	def.logf(format, args...)
 	os.Exit(1)
 }
 
 // Setup default logger with options
 func Setup(opts ...Option) {
 	def = New(opts...)
-	def.callerSkip = 2
 }
 
 // Default returns pre-constructed def logger (debug off, callers disabled)
diff --git a/backend/vendor/github.com/go-pkgz/lgr/logger.go b/backend/vendor/github.com/go-pkgz/lgr/logger.go
index d3346fd7..1f114eae 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/logger.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/logger.go
@@ -1,6 +1,7 @@
 package lgr
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -8,44 +9,94 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"text/template"
 	"time"
 )
 
-var levels = []string{"DEBUG", "INFO", "WARN", "ERROR", "PANIC", "FATAL"}
+var levels = []string{"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "PANIC", "FATAL"}
+
+const (
+	Short      = `{{.DT.Format "2006/01/02 15:04:05"}} {{.Level}} {{.Message}}`
+	WithMsec   = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} {{.Message}}`
+	WithPkg    = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerPkg}}) {{.Message}}`
+	ShortDebug = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFile}}:{{.CallerLine}}) {{.Message}}`
+	FuncDebug  = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFunc}}) {{.Message}}`
+	FullDebug  = `{{.DT.Format "2006/01/02 15:04:05.000"}} {{.Level}} ({{.CallerFile}}:{{.CallerLine}} {{.CallerFunc}}) {{.Message}}`
+)
 
 // Logger provided simple logger with basic support of levels. Thread safe
 type Logger struct {
-	stdout, stderr    io.Writer
-	dbg               bool
-	lock              sync.Mutex
-	callerFile        bool
-	callerFunc        bool
-	callerPkg         bool
-	callerSkip        int
-	ignoredPkgCallers []string
+	// set with Option calls
+	stdout, stderr io.Writer // destination writes for out and err
+	dbg            bool      // allows reporting for DEBUG level
+	trace          bool      // allows reporting for TRACE and DEBUG levels
+	callerFile     bool      // reports caller file, i.e. /go/src/github.com/go-pkgz/lgr/logger.go
+	callerFunc     bool      // reports caller function name, i.e. foo/bar.myFunc
+	callerPkg      bool      // reports caller package name
+	levelBraces    bool      // encloses level with [], i.e. [INFO]
+	callerDepth    int       // how many stack frames to skip
+	format         string    // layout template
 
-	now         nowFn
-	fatal       panicFn
-	levelBraces bool
-	msec        bool
+	// internal use
+	now           nowFn
+	fatal         panicFn
+	msec          bool
+	lock          sync.Mutex
+	callerOn      bool
+	levelBracesOn bool
+	templ         *template.Template
 }
 
+// can be redefined internally for testing
 type nowFn func() time.Time
 type panicFn func()
 
-// New makes new leveled logger. Accepts dbg flag turing on info about the caller and allowing DEBUG messages/
+type layout struct {
+	DT         time.Time
+	Level      string
+	Message    string
+	CallerPkg  string
+	CallerFile string
+	CallerFunc string
+	CallerLine int
+}
+
+// New makes new leveled logger. Accepts dbg flag turing on info about the caller and allowing DEBUG messages.
 // Two writers can be passed optionally - first for out and second for err
 func New(options ...Option) *Logger {
+
 	res := Logger{
-		now:        time.Now,
-		fatal:      func() { os.Exit(1) },
-		stdout:     os.Stdout,
-		stderr:     os.Stderr,
-		callerSkip: 1,
+		now:         time.Now,
+		fatal:       func() { os.Exit(1) },
+		stdout:      os.Stdout,
+		stderr:      os.Stderr,
+		callerDepth: 0,
 	}
 	for _, opt := range options {
 		opt(&res)
 	}
+
+	var err error
+	if res.format == "" {
+		res.format = res.templateFromOptions()
+	}
+
+	res.templ, err = template.New("lgr").Parse(res.format)
+	if err != nil {
+		fmt.Printf("invalid template %s, error %v. switched to %s\n", res.format, err, Short)
+		res.format = Short
+		res.templ = template.Must(template.New("lgrDefault").Parse(Short))
+	}
+
+	buf := bytes.Buffer{}
+	if err = res.templ.Execute(&buf, layout{}); err != nil {
+		fmt.Printf("failed to execute template %s, error %v. switched to %s\n", res.format, err, Short)
+		res.format = Short
+		res.templ = template.Must(template.New("lgrDefault").Parse(Short))
+	}
+
+	res.callerOn = strings.Contains(res.format, "{{.Caller")
+	res.levelBracesOn = strings.Contains(res.format, "[{{.Level}}]")
 	return &res
 }
 
@@ -54,92 +105,154 @@ func New(options ...Option) *Logger {
 // ERROR and FATAL also send the same line to err writer.
 // FATAL adds runtime stack and os.exit(1), like panic.
 func (l *Logger) Logf(format string, args ...interface{}) {
+	// to align call depth between (*Logger).Logf() and, for example, Printf()
+	l.logf(format, args...)
+}
 
-	// format timestamp with or without msecs
-	ts := func() (res string) {
-		if l.msec {
-			return l.now().Format("2006/01/02 15:04:05.000")
-		}
-		return l.now().Format("2006/01/02 15:04:05")
-	}
+func (l *Logger) logf(format string, args ...interface{}) {
 
 	lv, msg := l.extractLevel(fmt.Sprintf(format, args...))
 	if lv == "DEBUG" && !l.dbg {
 		return
 	}
-	var bld strings.Builder
-	bld.WriteString(ts())
-	bld.WriteString(l.formatLevel(lv))
-	bld.WriteString(" ")
-
-	if l.callerFile || l.callerFunc || l.callerPkg {
-		if pc, file, line, ok := runtime.Caller(l.callerSkip); ok {
-
-			funcName, fileInfo := "", ""
-
-			if l.callerFunc {
-				funcNameElems := strings.Split(runtime.FuncForPC(pc).Name(), "/")
-				funcName = funcNameElems[len(funcNameElems)-1]
-			}
-
-			if l.callerFile {
-				fnameElems := strings.Split(file, "/")
-				fileInfo = fmt.Sprintf("%s:%d", strings.Join(fnameElems[len(fnameElems)-2:], "/"), line)
-				if l.callerFunc {
-					fileInfo += " "
-				}
-			}
-			// callerPkg only if no other callers
-			if l.callerPkg && !l.callerFile && !l.callerFunc {
-				file = l.ignoreCaller(file)
-				_, fileInfo = path.Split(path.Dir(file))
-				if l.callerFunc {
-					fileInfo += " "
-				}
-			}
-			srcFileInfo := fmt.Sprintf("{%s%s} ", fileInfo, funcName)
-			bld.WriteString(srcFileInfo)
-		}
+	if lv == "TRACE" && !l.trace {
+		return
 	}
 
-	bld.WriteString(msg)  //nolint
-	bld.WriteString("\n") //nolint
+	ci := callerInfo{}
+	if l.callerOn { // optimization to avlod expensive caller evaluation if not in template
+		ci = l.reportCaller(l.callerDepth)
+	}
+
+	elems := layout{
+		DT:         l.now(),
+		Level:      l.formatLevel(lv),
+		Message:    strings.TrimSuffix(msg, "\n"),
+		CallerFunc: ci.FuncName,
+		CallerFile: ci.File,
+		CallerPkg:  ci.Pkg,
+		CallerLine: ci.Line,
+	}
+
+	buf := bytes.Buffer{}
+	err := l.templ.Execute(&buf, elems) // once constructed, a template may be executed safely in parallel.
+	if err != nil {
+		fmt.Printf("failed to execute template, %v\n", err)
+	}
+	buf.WriteString("\n")
+
+	data := buf.Bytes()
+	if l.levelBracesOn {
+		data = bytes.Replace(data, []byte("[WARN ]"), []byte("[WARN] "), 1)
+		data = bytes.Replace(data, []byte("[INFO ]"), []byte("[INFO] "), 1)
+	}
 
 	l.lock.Lock()
-	msgb := []byte(bld.String())
-	l.stdout.Write(msgb) //nolint
+	_, _ = l.stdout.Write(data)
 
+	// write to err as well for high levels
 	switch lv {
 	case "PANIC", "FATAL":
-		l.stderr.Write(msgb)      //nolint
-		bld.WriteString("\n")     //nolint
-		l.stderr.Write(getDump()) //nolint
+		_, _ = l.stderr.Write(data)
+		_, _ = l.stderr.Write(getDump())
 		l.fatal()
 	case "ERROR":
-		l.stderr.Write(msgb) //nolint
+		_, _ = l.stderr.Write(data)
 	}
 
 	l.lock.Unlock()
 }
 
-func (l *Logger) ignoreCaller(p string) string {
-	for _, s := range l.ignoredPkgCallers {
-		if strings.Contains(p, "/"+s+"/") {
-			return strings.Replace(p, "/"+s, "", 1)
-		}
-	}
-	return p
+type callerInfo struct {
+	File     string
+	Line     int
+	FuncName string
+	Pkg      string
 }
 
-func (l *Logger) formatLevel(lv string) string {
+// calldepth 0 identifying the caller of reportCaller()
+func (l *Logger) reportCaller(calldepth int) (res callerInfo) {
 
-	brace := func(b string) string {
-		if l.levelBraces {
-			return b
+	// caller gets file, line number abd function name via runtime.Callers
+	// file looks like /go/src/github.com/go-pkgz/lgr/logger.go
+	// file is an empty string if not known.
+	// funcName looks like:
+	//   main.Test
+	//   foo/bar.Test
+	//   foo/bar.Test.func1
+	//   foo/bar.(*Bar).Test
+	//   foo/bar.glob..func1
+	// funcName is an empty string if not known.
+	// line is a zero if not known.
+	caller := func(calldepth int) (file string, line int, funcName string) {
+		pcs := make([]uintptr, 1)
+		n := runtime.Callers(calldepth, pcs)
+		if n != 1 {
+			return "", 0, ""
 		}
-		return ""
+
+		frame, _ := runtime.CallersFrames(pcs).Next()
+
+		return frame.File, frame.Line, frame.Function
 	}
 
+	// add 5 to adjust stack level because it was called from 3 nested functions added by lgr, i.e. caller,
+	// reportCaller and logf, plus 2 frames by runtime
+	filePath, line, funcName := caller(calldepth + 2 + 3)
+	if (filePath == "") || (line <= 0) || (funcName == "") {
+		return callerInfo{}
+	}
+
+	_, pkgInfo := path.Split(path.Dir(filePath))
+	res.Pkg = pkgInfo
+
+	res.File = filePath
+	if pathElems := strings.Split(filePath, "/"); len(pathElems) > 2 {
+		res.File = strings.Join(pathElems[len(pathElems)-2:], "/")
+	}
+	res.Line = line
+
+	funcNameElems := strings.Split(funcName, "/")
+	res.FuncName = funcNameElems[len(funcNameElems)-1]
+
+	return res
+}
+
+// make template from options flag
+func (l *Logger) templateFromOptions() (res string) {
+
+	orElse := func(flag bool, value string, elseValue string) string {
+		if flag {
+			return value
+		}
+		return elseValue
+	}
+
+	var parts []string
+
+	parts = append(parts, orElse(l.msec, `{{.DT.Format "2006/01/02 15:04:05.000"}}`, `{{.DT.Format "2006/01/02 15:04:05"}}`))
+	parts = append(parts, orElse(l.levelBraces, `[{{.Level}}]`, `{{.Level}}`))
+
+	if l.callerFile || l.callerFunc || l.callerPkg {
+		var callerParts []string
+		if v := orElse(l.callerFile, `{{.CallerFile}}:{{.CallerLine}}`, ""); v != "" {
+			callerParts = append(callerParts, v)
+		}
+		if v := orElse(l.callerFunc, `{{.CallerFunc}}`, ""); v != "" {
+			callerParts = append(callerParts, v)
+		}
+		if v := orElse(l.callerPkg, `{{.CallerPkg}}`, ""); v != "" {
+			callerParts = append(callerParts, v)
+		}
+		parts = append(parts, "("+strings.Join(callerParts, " ")+")")
+	}
+	parts = append(parts, "{{.Message}}")
+	return strings.Join(parts, " ")
+}
+
+// formatLevel aligns level to 5 chars
+func (l *Logger) formatLevel(lv string) string {
+
 	if lv == "" {
 		return ""
 	}
@@ -148,9 +261,10 @@ func (l *Logger) formatLevel(lv string) string {
 	if len(lv) == 4 {
 		spaces = " "
 	}
-	return " " + brace("[") + lv + brace("]") + spaces
+	return lv + spaces
 }
 
+// extractLevel parses messages with optional level prefix and returns level and the message with stripped level
 func (l *Logger) extractLevel(line string) (level, msg string) {
 	for _, lv := range levels {
 		if strings.HasPrefix(line, lv) {
@@ -196,9 +310,23 @@ func Debug(l *Logger) {
 	l.dbg = true
 }
 
-// CallerFile adds caller info with file, and line number
-func CallerFile(l *Logger) {
-	l.callerFile = true
+// Trace turn on trace + dbg mode
+func Trace(l *Logger) {
+	l.dbg = true
+	l.trace = true
+}
+
+// CallerDepth sets number of stack frame skipped for caller reporting
+func CallerDepth(n int) Option {
+	return func(l *Logger) {
+		l.callerDepth = n
+	}
+}
+
+func Format(f string) Option {
+	return func(l *Logger) {
+		l.format = f
+	}
 }
 
 // CallerFunc adds caller info with function name
@@ -211,26 +339,16 @@ func CallerPkg(l *Logger) {
 	l.callerPkg = true
 }
 
-// CallerIgnore sets packages skipped from logging caller
-func CallerIgnore(ignores ...string) Option {
-	return func(l *Logger) {
-		l.ignoredPkgCallers = ignores
-	}
-}
-
-// CallerSkip sets how many trace levels to skip.
-// by default this value is 1 , i.e. skip logger level only
-func CallerSkip(n int) Option {
-	return func(l *Logger) {
-		l.callerSkip = n
-	}
-}
-
 // LevelBraces adds [] to level
 func LevelBraces(l *Logger) {
 	l.levelBraces = true
 }
 
+// CallerFile adds caller info with file, and line number
+func CallerFile(l *Logger) {
+	l.callerFile = true
+}
+
 // Msec adds .msec to timestamp
 func Msec(l *Logger) {
 	l.msec = true
diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt
index 60f43d75..251131d9 100644
--- a/backend/vendor/modules.txt
+++ b/backend/vendor/modules.txt
@@ -39,7 +39,7 @@ github.com/go-pkgz/auth/logger
 github.com/go-pkgz/auth/middleware
 # github.com/go-pkgz/lcw v0.2.0
 github.com/go-pkgz/lcw
-# github.com/go-pkgz/lgr v0.4.0
+# github.com/go-pkgz/lgr v0.6.0
 github.com/go-pkgz/lgr
 # github.com/go-pkgz/mongo v1.1.2
 github.com/go-pkgz/mongo

From 45ae2e6f130e3ef49b747ae0b6d54782c6fb939e Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Sun, 31 Mar 2019 16:21:32 -0500
Subject: [PATCH 34/45] revendor with lgr 0.6.1

---
 backend/app/main.go                           |  4 +-
 backend/go.mod                                |  2 +-
 backend/go.sum                                |  2 +
 .../vendor/github.com/go-pkgz/lgr/README.md   |  6 +-
 .../vendor/github.com/go-pkgz/lgr/logger.go   | 65 ++++++++++++-------
 backend/vendor/modules.txt                    |  2 +-
 6 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/backend/app/main.go b/backend/app/main.go
index b23bf9d3..4e41a3d6 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -62,10 +62,10 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.Msec)
 		return
 	}
-	log.Setup(log.Msec, log.LevelBraces, log.CallerPkg)
+	log.Setup(log.Msec, log.CallerPkg)
 }
 
 // getDump reads runtime stack and returns as a string
diff --git a/backend/go.mod b/backend/go.mod
index 49c76116..173adbcc 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/go-chi/render v1.0.0
 	github.com/go-pkgz/auth v0.5.0
 	github.com/go-pkgz/lcw v0.2.0
-	github.com/go-pkgz/lgr v0.6.0
+	github.com/go-pkgz/lgr v0.6.1
 	github.com/go-pkgz/mongo v1.1.2
 	github.com/go-pkgz/repeater v1.1.1
 	github.com/go-pkgz/rest v1.4.0
diff --git a/backend/go.sum b/backend/go.sum
index d5d43b52..25cdfbdb 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -33,6 +33,8 @@ github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/
 github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/lgr v0.6.0 h1:Z9FRhfSyuASiF05iXRj+clTAAl8+1EfLR7SeyLIaGKg=
 github.com/go-pkgz/lgr v0.6.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.1 h1:poohUbv/iguoQ6bzJ5j/Ubl1VcsjU+rzTbbxsKbkGXk=
+github.com/go-pkgz/lgr v0.6.1/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
 github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
 github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
diff --git a/backend/vendor/github.com/go-pkgz/lgr/README.md b/backend/vendor/github.com/go-pkgz/lgr/README.md
index de7aae27..77d81424 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/README.md
+++ b/backend/vendor/github.com/go-pkgz/lgr/README.md
@@ -7,14 +7,14 @@
 ## usage
 
 ```go
-    l := lgr.New(lgr.Debug, lgr.CallerFile) // allow debug and caller file info
+    l := lgr.New(lgr.Msec, lgr.Debug, lgr.CallerFile, lgr.CallerFunc) // allow debug and caller info, timestamp with milliseconds
     l.Logf("INFO some important message, %v", err)
     l.Logf("DEBUG some less important message, %v", err)
 ```
 
 output looks like this:
 ```
-2018/01/07 13:02:34.000 INFO  {svc/handler.go:101 h.MyFunc1} some important message, can't open file`
+2018/01/07 13:02:34.000 INFO  {svc/handler.go:101 h.MyFunc1} some important message, can't open file myfile.xyz
 2018/01/07 13:02:34.015 DEBUG {svc/handler.go:155 h.MyFunc2} some less important message, file is too small`
 ```
 
@@ -25,7 +25,7 @@ _Without `lgr.Caller*` it will drop `{caller}` part_
 ### interfaces and default loggers
 
 - `lgr` package provides a single interface `lgr.L` with a single method `Logf(format string, args ...interface{})`. Function wrapper `lgr.Func` allows to make `lgr.L` from a function directly.
-- Default logger functionality can be used without `lgr.New`, but just `lgr.Printf`
+- Default logger functionality can be used without `lgr.New` (see "global logger")
 - Two predefined loggers available: `lgr.NoOp` (do-nothing logger) and `lgr.Std` (passing directly to stdlib log)
 
 ### options
diff --git a/backend/vendor/github.com/go-pkgz/lgr/logger.go b/backend/vendor/github.com/go-pkgz/lgr/logger.go
index 1f114eae..264c9b2e 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/logger.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/logger.go
@@ -1,3 +1,11 @@
+// Package lgr provides a simple logger with some extras. Primary way to log is Logf method.
+// The logger's output can be customized in 2 ways:
+//   - by passing formatting template, i.e. lgr.New(lgr.Format(lgr.Short))
+//   - by setting individual formatting flags, i.e. lgr.New(lgr.Msec, lgr.CallerFunc)
+// Leveled output works for messages based on level prefix, i.e. Logf("INFO some message") means INFO level.
+// Debug and trace levels can be filtered based on lgr.Trace and lgr.Debug options.
+// ERROR, FATAL and PANIC levels send to err as well. Both FATAL and PANIC also print stack trace and terminate caller application with os.Exit(1)
+
 package lgr
 
 import (
@@ -30,11 +38,11 @@ type Logger struct {
 	stdout, stderr io.Writer // destination writes for out and err
 	dbg            bool      // allows reporting for DEBUG level
 	trace          bool      // allows reporting for TRACE and DEBUG levels
-	callerFile     bool      // reports caller file, i.e. /go/src/github.com/go-pkgz/lgr/logger.go
-	callerFunc     bool      // reports caller function name, i.e. foo/bar.myFunc
+	callerFile     bool      // reports caller file with line number, i.e. foo/bar.go:89
+	callerFunc     bool      // reports caller function name, i.e. bar.myFunc
 	callerPkg      bool      // reports caller package name
 	levelBraces    bool      // encloses level with [], i.e. [INFO]
-	callerDepth    int       // how many stack frames to skip
+	callerDepth    int       // how many stack frames to skip, relative to the real (reported) frame
 	format         string    // layout template
 
 	// internal use
@@ -51,6 +59,7 @@ type Logger struct {
 type nowFn func() time.Time
 type panicFn func()
 
+// layout holds all parts to construct the final message with template
 type layout struct {
 	DT         time.Time
 	Level      string
@@ -61,8 +70,8 @@ type layout struct {
 	CallerLine int
 }
 
-// New makes new leveled logger. Accepts dbg flag turing on info about the caller and allowing DEBUG messages.
-// Two writers can be passed optionally - first for out and second for err
+// New makes new leveled logger. By default writes to stdout/stderr.
+// default format: 2018/01/07 13:02:34.123 DEBUG some message 123
 func New(options ...Option) *Logger {
 
 	res := Logger{
@@ -101,9 +110,9 @@ func New(options ...Option) *Logger {
 }
 
 // Logf implements L interface to output with printf style.
-// Each line prefixed with ts, level and optionally (dbg mode only) by caller info.
+// DEBUG and TRACE filtered out by dbg and trace flags.
 // ERROR and FATAL also send the same line to err writer.
-// FATAL adds runtime stack and os.exit(1), like panic.
+// FATAL and PANIC adds runtime stack and os.exit(1), like panic.
 func (l *Logger) Logf(format string, args ...interface{}) {
 	// to align call depth between (*Logger).Logf() and, for example, Printf()
 	l.logf(format, args...)
@@ -120,7 +129,7 @@ func (l *Logger) logf(format string, args ...interface{}) {
 	}
 
 	ci := callerInfo{}
-	if l.callerOn { // optimization to avlod expensive caller evaluation if not in template
+	if l.callerOn { // optimization to avoid expensive caller evaluation if caller info not in the template
 		ci = l.reportCaller(l.callerDepth)
 	}
 
@@ -142,7 +151,7 @@ func (l *Logger) logf(format string, args ...interface{}) {
 	buf.WriteString("\n")
 
 	data := buf.Bytes()
-	if l.levelBracesOn {
+	if l.levelBracesOn { // rearrange space in short levels
 		data = bytes.Replace(data, []byte("[WARN ]"), []byte("[WARN] "), 1)
 		data = bytes.Replace(data, []byte("[INFO ]"), []byte("[INFO] "), 1)
 	}
@@ -150,14 +159,17 @@ func (l *Logger) logf(format string, args ...interface{}) {
 	l.lock.Lock()
 	_, _ = l.stdout.Write(data)
 
-	// write to err as well for high levels
+	// write to err as well for high levels, exit(1) on fatal and panic and dump stack on panic level
 	switch lv {
-	case "PANIC", "FATAL":
+	case "ERROR":
+		_, _ = l.stderr.Write(data)
+	case "FATAL":
+		_, _ = l.stderr.Write(data)
+		l.fatal()
+	case "PANIC":
 		_, _ = l.stderr.Write(data)
 		_, _ = l.stderr.Write(getDump())
 		l.fatal()
-	case "ERROR":
-		_, _ = l.stderr.Write(data)
 	}
 
 	l.lock.Unlock()
@@ -218,9 +230,15 @@ func (l *Logger) reportCaller(calldepth int) (res callerInfo) {
 	return res
 }
 
-// make template from options flag
+// make template from option flags
 func (l *Logger) templateFromOptions() (res string) {
 
+	const (
+		// escape { and } from templates to allow "{some/blah}" output for caller
+		openCallerBrace  = `{{"{"}}`
+		closeCallerBrace = `{{"}"}}`
+	)
+
 	orElse := func(flag bool, value string, elseValue string) string {
 		if flag {
 			return value
@@ -244,7 +262,7 @@ func (l *Logger) templateFromOptions() (res string) {
 		if v := orElse(l.callerPkg, `{{.CallerPkg}}`, ""); v != "" {
 			callerParts = append(callerParts, v)
 		}
-		parts = append(parts, "("+strings.Join(callerParts, " ")+")")
+		parts = append(parts, openCallerBrace+strings.Join(callerParts, " ")+closeCallerBrace)
 	}
 	parts = append(parts, "{{.Message}}")
 	return strings.Join(parts, " ")
@@ -291,14 +309,14 @@ func getDump() []byte {
 // Option func type
 type Option func(l *Logger)
 
-// Out sets out writer
+// Out sets out writer, stdout by default
 func Out(w io.Writer) Option {
 	return func(l *Logger) {
 		l.stdout = w
 	}
 }
 
-// Err sets error writer
+// Err sets error writer, stderr by default
 func Err(w io.Writer) Option {
 	return func(l *Logger) {
 		l.stderr = w
@@ -316,40 +334,41 @@ func Trace(l *Logger) {
 	l.trace = true
 }
 
-// CallerDepth sets number of stack frame skipped for caller reporting
+// CallerDepth sets number of stack frame skipped for caller reporting, 0 by default
 func CallerDepth(n int) Option {
 	return func(l *Logger) {
 		l.callerDepth = n
 	}
 }
 
+// Format sets output layout, overwrites all options for individual parts, i.e. Caller*, Msec and LevelBraces
 func Format(f string) Option {
 	return func(l *Logger) {
 		l.format = f
 	}
 }
 
-// CallerFunc adds caller info with function name
+// CallerFunc adds caller info with function name. Ignored if Format option used.
 func CallerFunc(l *Logger) {
 	l.callerFunc = true
 }
 
-// CallerPkg adds caller's package name
+// CallerPkg adds caller's package name. Ignored if Format option used.
 func CallerPkg(l *Logger) {
 	l.callerPkg = true
 }
 
-// LevelBraces adds [] to level
+// LevelBraces surrounds level with [], i.e. [INFO]. Ignored if Format option used.
 func LevelBraces(l *Logger) {
 	l.levelBraces = true
 }
 
-// CallerFile adds caller info with file, and line number
+// CallerFile adds caller info with file, and line number. Ignored if Format option used.
 func CallerFile(l *Logger) {
 	l.callerFile = true
 }
 
-// Msec adds .msec to timestamp
+// Msec adds .msec to timestamp. Ignored if Format option used.
 func Msec(l *Logger) {
 	l.msec = true
 }
diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt
index 251131d9..bdc6b4eb 100644
--- a/backend/vendor/modules.txt
+++ b/backend/vendor/modules.txt
@@ -39,7 +39,7 @@ github.com/go-pkgz/auth/logger
 github.com/go-pkgz/auth/middleware
 # github.com/go-pkgz/lcw v0.2.0
 github.com/go-pkgz/lcw
-# github.com/go-pkgz/lgr v0.6.0
+# github.com/go-pkgz/lgr v0.6.1
 github.com/go-pkgz/lgr
 # github.com/go-pkgz/mongo v1.1.2
 github.com/go-pkgz/mongo

From 0f433722da06b5d6e3e7a619daef332e674e0916 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Thu, 4 Apr 2019 01:53:47 -0500
Subject: [PATCH 35/45] disable picture upload for anonymous

---
 backend/app/cmd/server.go           | 1 +
 backend/app/rest/api/rest.go        | 2 +-
 backend/app/store/image/fs_store.go | 5 +++++
 backend/go.mod                      | 2 +-
 backend/go.sum                      | 2 ++
 5 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go
index 5b88d957..160aec66 100644
--- a/backend/app/cmd/server.go
+++ b/backend/app/cmd/server.go
@@ -525,6 +525,7 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 		providers++
 	}
 	if s.Auth.Dev {
+		log.Print("[INFO] dev access enabled")
 		authenticator.AddProvider("dev", "", "")
 		providers++
 	}
diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index f02b7361..86acf991 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -273,7 +273,7 @@ func (s *Rest) routes() chi.Router {
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
 			rauth.Use(authMiddleware.Auth)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-			rauth.Post("/picture", s.savePictureCtrl)
+			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
 		})
 
 	})
diff --git a/backend/app/store/image/fs_store.go b/backend/app/store/image/fs_store.go
index 79049635..cc99aec7 100644
--- a/backend/app/store/image/fs_store.go
+++ b/backend/app/store/image/fs_store.go
@@ -114,6 +114,11 @@ func (f *FileSystem) Load(id string) (io.ReadCloser, int64, error) {
 
 // Cleanup runs scan of staging and removes old files based on ttl
 func (f *FileSystem) Cleanup(ctx context.Context, ttl time.Duration) error {
+
+	if _, err := os.Stat(f.Staging); os.IsNotExist(err) {
+		return nil
+	}
+
 	err := filepath.Walk(f.Staging, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
diff --git a/backend/go.mod b/backend/go.mod
index 173adbcc..3fa72690 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/go-chi/render v1.0.0
 	github.com/go-pkgz/auth v0.5.0
 	github.com/go-pkgz/lcw v0.2.0
-	github.com/go-pkgz/lgr v0.6.1
+	github.com/go-pkgz/lgr v0.6.2
 	github.com/go-pkgz/mongo v1.1.2
 	github.com/go-pkgz/repeater v1.1.1
 	github.com/go-pkgz/rest v1.4.0
diff --git a/backend/go.sum b/backend/go.sum
index 25cdfbdb..1a75e68e 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -35,6 +35,8 @@ github.com/go-pkgz/lgr v0.6.0 h1:Z9FRhfSyuASiF05iXRj+clTAAl8+1EfLR7SeyLIaGKg=
 github.com/go-pkgz/lgr v0.6.0/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/lgr v0.6.1 h1:poohUbv/iguoQ6bzJ5j/Ubl1VcsjU+rzTbbxsKbkGXk=
 github.com/go-pkgz/lgr v0.6.1/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
+github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
+github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
 github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
 github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
 github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=

From 70134578e18c9f00b22a69a4b14f3b9f1d141511 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Thu, 4 Apr 2019 02:17:12 -0500
Subject: [PATCH 36/45] switch to chi v4

---
 backend/app/rest/api/rest.go                  |   2 +-
 backend/go.mod                                |   2 +-
 backend/go.sum                                |   2 +
 .../vendor/github.com/go-chi/chi/.travis.yml  |  18 +-
 .../vendor/github.com/go-chi/chi/CHANGELOG.md |  23 ++
 .../vendor/github.com/go-chi/chi/README.md    |  59 ++--
 .../vendor/github.com/go-chi/chi/context.go   |  14 +-
 .../go-chi/chi/middleware/closenotify17.go    |  42 ---
 .../go-chi/chi/middleware/closenotify18.go    |  17 --
 .../go-chi/chi/middleware/compress.go         | 271 +++++++++++-------
 .../go-chi/chi/middleware/compress18.go       |  15 -
 .../go-chi/chi/middleware/content_type.go     |   6 +
 .../go-chi/chi/middleware/logger.go           |  40 +--
 .../go-chi/chi/middleware/nocache.go          |   2 +-
 .../go-chi/chi/middleware/realip.go           |   2 +-
 .../go-chi/chi/middleware/request_id.go       |  10 +-
 .../github.com/go-chi/chi/middleware/strip.go |  10 +-
 .../go-chi/chi/middleware/terminal.go         |   6 +-
 .../go-chi/chi/middleware/timeout.go          |   3 +-
 .../go-chi/chi/middleware/wrap_writer.go      |  59 +++-
 .../go-chi/chi/middleware/wrap_writer17.go    |  34 ---
 .../go-chi/chi/middleware/wrap_writer18.go    |  41 ---
 backend/vendor/github.com/go-chi/chi/mux.go   |   3 +-
 backend/vendor/github.com/go-chi/chi/tree.go  |  22 +-
 .../github.com/go-pkgz/lgr/.golangci.yml      |  60 ++++
 .../vendor/github.com/go-pkgz/lgr/.travis.yml |   5 +-
 .../vendor/github.com/go-pkgz/lgr/README.md   |  18 +-
 .../vendor/github.com/go-pkgz/lgr/adaptor.go  |   4 +-
 .../github.com/go-pkgz/lgr/interface.go       |   3 +-
 .../vendor/github.com/go-pkgz/lgr/logger.go   | 181 ++++--------
 .../vendor/github.com/go-pkgz/lgr/options.go  |  70 +++++
 backend/vendor/modules.txt                    |   4 +-
 32 files changed, 558 insertions(+), 490 deletions(-)
 delete mode 100644 backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go
 delete mode 100644 backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go
 delete mode 100644 backend/vendor/github.com/go-chi/chi/middleware/compress18.go
 delete mode 100644 backend/vendor/github.com/go-chi/chi/middleware/wrap_writer17.go
 delete mode 100644 backend/vendor/github.com/go-chi/chi/middleware/wrap_writer18.go
 create mode 100644 backend/vendor/github.com/go-pkgz/lgr/.golangci.yml
 create mode 100644 backend/vendor/github.com/go-pkgz/lgr/options.go

diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go
index 86acf991..e855b942 100644
--- a/backend/app/rest/api/rest.go
+++ b/backend/app/rest/api/rest.go
@@ -262,7 +262,7 @@ func (s *Rest) routes() chi.Router {
 			rauth.Put("/comment/{id}", s.updateCommentCtrl)
 			rauth.Post("/comment", s.createCommentCtrl)
 			rauth.With(rejectAnonUser).Put("/vote/{id}", s.voteCtrl)
-			rauth.Post("/deleteme", s.deleteMeCtrl)
+			rauth.With(rejectAnonUser).Post("/deleteme", s.deleteMeCtrl)
 		})
 
 		rapi.Group(func(rauth chi.Router) {
diff --git a/backend/go.mod b/backend/go.mod
index 3fa72690..577d345e 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/didip/tollbooth v4.0.0+incompatible
 	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
 	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
-	github.com/go-chi/chi v3.3.2+incompatible
+	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/go-chi/cors v1.0.0
 	github.com/go-chi/render v1.0.0
 	github.com/go-pkgz/auth v0.5.0
diff --git a/backend/go.sum b/backend/go.sum
index 1a75e68e..ee3962c0 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -22,6 +22,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7a
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-chi/chi v3.3.2+incompatible h1:uQNcQN3NsV1j4ANsPh42P4ew4t6rnRbJb8frvpp31qQ=
 github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
+github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
 github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
diff --git a/backend/vendor/github.com/go-chi/chi/.travis.yml b/backend/vendor/github.com/go-chi/chi/.travis.yml
index a6d5de85..de3287e1 100644
--- a/backend/vendor/github.com/go-chi/chi/.travis.yml
+++ b/backend/vendor/github.com/go-chi/chi/.travis.yml
@@ -1,18 +1,18 @@
 language: go
 
 go:
-  - 1.7.x
-  - 1.8.x
-  - 1.9.x
-
-install:
-  - go get -u golang.org/x/tools/cmd/goimports
-  - go get -u github.com/golang/lint/golint
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
 
 script:
   - go get -d -t ./...
   - go vet ./...
-  - golint ./...
   - go test ./...
   - >
-    goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :
+    go_version=$(go version);
+    if [ ${go_version:13:4} = "1.12" ]; then
+      go get -u golang.org/x/tools/cmd/goimports;
+      goimports -d -e ./ | grep '.*' && { echo; echo "Aborting due to non-empty goimports output."; exit 1; } || :;
+    fi
+
diff --git a/backend/vendor/github.com/go-chi/chi/CHANGELOG.md b/backend/vendor/github.com/go-chi/chi/CHANGELOG.md
index 5f0ab254..d03e40c6 100644
--- a/backend/vendor/github.com/go-chi/chi/CHANGELOG.md
+++ b/backend/vendor/github.com/go-chi/chi/CHANGELOG.md
@@ -1,5 +1,28 @@
 # Changelog
 
+## v4.0.0 (2019-01-10)
+
+- chi v4 requires Go 1.10.3+ (or Go 1.9.7+) - we have deprecated support for Go 1.7 and 1.8
+- router: respond with 404 on router with no routes (#362)
+- router: additional check to ensure wildcard is at the end of a url pattern (#333)
+- middleware: deprecate use of http.CloseNotifier (#347)
+- middleware: fix RedirectSlashes to include query params on redirect (#334)
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.4...v4.0.0
+
+
+## v3.3.4 (2019-01-07)
+
+- Minor middleware improvements. No changes to core library/router. Moving v3 into its
+- own branch as a version of chi for Go 1.7, 1.8, 1.9, 1.10, 1.11
+- History of changes: see https://github.com/go-chi/chi/compare/v3.3.3...v3.3.4
+
+
+## v3.3.3 (2018-08-27)
+
+- Minor release
+- See https://github.com/go-chi/chi/compare/v3.3.2...v3.3.3
+
+
 ## v3.3.2 (2017-12-22)
 
 - Support to route trailing slashes on mounted sub-routers (#281)
diff --git a/backend/vendor/github.com/go-chi/chi/README.md b/backend/vendor/github.com/go-chi/chi/README.md
index c71a3a01..d36d4db5 100644
--- a/backend/vendor/github.com/go-chi/chi/README.md
+++ b/backend/vendor/github.com/go-chi/chi/README.md
@@ -3,7 +3,7 @@
 
 [![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis]
 
-`chi` is a lightweight, idiomatic and composable router for building Go 1.7+ HTTP services. It's
+`chi` is a lightweight, idiomatic and composable router for building Go HTTP services. It's
 especially good at helping you write large REST API services that are kept maintainable as your
 project grows and changes. `chi` is built on the new `context` package introduced in Go 1.7 to
 handle signaling, cancelation and request-scoped values across a handler chain.
@@ -31,18 +31,12 @@ included some useful/optional subpackages: [middleware](/middleware), [render](h
 * **Context control** - built on new `context` package, providing value chaining, cancelations and timeouts
 * **Robust** - in production at Pressly, CloudFlare, Heroku, 99Designs, and many others (see [discussion](https://github.com/go-chi/chi/issues/91))
 * **Doc generation** - `docgen` auto-generates routing documentation from your source to JSON or Markdown
-* **No external dependencies** - plain ol' Go 1.7+ stdlib + net/http
+* **No external dependencies** - plain ol' Go stdlib + net/http
 
 
 ## Examples
 
-* [rest](https://github.com/go-chi/chi/blob/master/_examples/rest/main.go) - REST APIs made easy, productive and maintainable
-* [logging](https://github.com/go-chi/chi/blob/master/_examples/logging/main.go) - Easy structured logging for any backend
-* [limits](https://github.com/go-chi/chi/blob/master/_examples/limits/main.go) - Timeouts and Throttling
-* [todos-resource](https://github.com/go-chi/chi/blob/master/_examples/todos-resource/main.go) - Struct routers/handlers, an example of another code layout style
-* [versions](https://github.com/go-chi/chi/blob/master/_examples/versions/main.go) - Demo of `chi/render` subpkg
-* [fileserver](https://github.com/go-chi/chi/blob/master/_examples/fileserver/main.go) - Easily serve static files
-* [graceful](https://github.com/go-chi/chi/blob/master/_examples/graceful/main.go) - Graceful context signaling and server shutdown
+See [_examples/](https://github.com/go-chi/chi/blob/master/_examples/) for a variety of examples.
 
 
 **As easy as:**
@@ -70,8 +64,8 @@ Here is a little preview of how routing looks like with chi. Also take a look at
 in JSON ([routes.json](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.json)) and in
 Markdown ([routes.md](https://github.com/go-chi/chi/blob/master/_examples/rest/routes.md)).
 
-I highly recommend reading the source of the [examples](#examples) listed above, they will show you all the features
-of chi and serve as a good form of documentation.
+I highly recommend reading the source of the [examples](https://github.com/go-chi/chi/blob/master/_examples/) listed
+above, they will show you all the features of chi and serve as a good form of documentation.
 
 ```go
 import (
@@ -232,7 +226,7 @@ type Router interface {
 }
 
 // Routes interface adds two methods for router traversal, which is also
-// used by the `docgen` subpackage to generation documentation for Routers.
+// used by the github.com/go-chi/docgen package to generate documentation for Routers.
 type Routes interface {
 	// Routes returns the routing tree in an easily traversable structure.
 	Routes() []Route
@@ -261,7 +255,7 @@ friendly with any middleware in the community. This offers much better extensibi
 of packages and is at the heart of chi's purpose.
 
 Here is an example of a standard net/http middleware handler using the new request context
-available in Go 1.7+. This middleware sets a hypothetical user identifier on the request
+available in Go. This middleware sets a hypothetical user identifier on the request
 context and calls the next handler in the chain.
 
 ```go
@@ -347,6 +341,7 @@ Please see https://github.com/go-chi for additional packages.
 | package                                            | description                                                 |
 |:---------------------------------------------------|:-------------------------------------------------------------
 | [cors](https://github.com/go-chi/cors)             | Cross-origin resource sharing (CORS)                        |
+| [docgen](https://github.com/go-chi/docgen)         | Print chi.Router routes at runtime                          |
 | [jwtauth](https://github.com/go-chi/jwtauth)       | JWT authentication                                          |
 | [hostrouter](https://github.com/go-chi/hostrouter) | Domain/host based request routing                           |
 | [httpcoala](https://github.com/go-chi/httpcoala)   | HTTP request coalescer                                      |
@@ -374,33 +369,33 @@ and..
 
 The benchmark suite: https://github.com/pkieltyka/go-http-routing-benchmark
 
-Results as of Aug 31, 2017 on Go 1.9.0
+Results as of Jan 9, 2019 with Go 1.11.4 on Linux X1 Carbon laptop
 
 ```shell
-BenchmarkChi_Param        	 3000000	       607 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Param5       	 2000000	       935 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Param20      	 1000000	      1944 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParamWrite   	 2000000	       664 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubStatic 	 2000000	       627 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubParam  	 2000000	       847 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GithubAll    	   10000	    175556 ns/op	   87700 B/op	     609 allocs/op
-BenchmarkChi_GPlusStatic  	 3000000	       566 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlusParam   	 2000000	       652 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlus2Params 	 2000000	       767 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_GPlusAll     	  200000	      9794 ns/op	    5616 B/op	      39 allocs/op
-BenchmarkChi_ParseStatic  	 3000000	       590 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParseParam   	 2000000	       656 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_Parse2Params 	 2000000	       715 ns/op	     432 B/op	       3 allocs/op
-BenchmarkChi_ParseAll     	  100000	     18045 ns/op	   11232 B/op	      78 allocs/op
-BenchmarkChi_StaticAll    	   10000	    108871 ns/op	   67827 B/op	     471 allocs/op
+BenchmarkChi_Param            3000000         475 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Param5           2000000         696 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Param20          1000000        1275 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParamWrite       3000000         505 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubStatic     3000000         508 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubParam      2000000         669 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GithubAll          10000      134627 ns/op     87699 B/op    609 allocs/op
+BenchmarkChi_GPlusStatic      3000000         402 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlusParam       3000000         500 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlus2Params     3000000         586 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_GPlusAll          200000        7237 ns/op      5616 B/op     39 allocs/op
+BenchmarkChi_ParseStatic      3000000         408 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParseParam       3000000         488 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_Parse2Params     3000000         551 ns/op       432 B/op      3 allocs/op
+BenchmarkChi_ParseAll          100000       13508 ns/op     11232 B/op     78 allocs/op
+BenchmarkChi_StaticAll          20000       81933 ns/op     67826 B/op    471 allocs/op
 ```
 
-Comparison with other routers: https://gist.github.com/pkieltyka/c089f309abeb179cfc4deaa519956d8c
+Comparison with other routers: https://gist.github.com/pkieltyka/123032f12052520aaccab752bd3e78cc
 
 NOTE: the allocs in the benchmark above are from the calls to http.Request's
 `WithContext(context.Context)` method that clones the http.Request, sets the `Context()`
 on the duplicated (alloc'd) request and returns it the new request object. This is just
-how setting context on a request in Go 1.7+ works.
+how setting context on a request in Go works.
 
 
 ## Credits
diff --git a/backend/vendor/github.com/go-chi/chi/context.go b/backend/vendor/github.com/go-chi/chi/context.go
index 30c5afed..229c9cbf 100644
--- a/backend/vendor/github.com/go-chi/chi/context.go
+++ b/backend/vendor/github.com/go-chi/chi/context.go
@@ -84,13 +84,13 @@ func (x *Context) URLParam(key string) string {
 //
 // For example,
 //
-// func Instrument(next http.Handler) http.Handler {
-//   return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-//     next.ServeHTTP(w, r)
-//     routePattern := chi.RouteContext(r.Context()).RoutePattern()
-//     measure(w, r, routePattern)
-// 	 })
-// }
+//   func Instrument(next http.Handler) http.Handler {
+//     return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+//       next.ServeHTTP(w, r)
+//       routePattern := chi.RouteContext(r.Context()).RoutePattern()
+//       measure(w, r, routePattern)
+//   	 })
+//   }
 func (x *Context) RoutePattern() string {
 	routePattern := strings.Join(x.RoutePatterns, "")
 	return strings.Replace(routePattern, "/*/", "/", -1)
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go b/backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go
deleted file mode 100644
index 95802b13..00000000
--- a/backend/vendor/github.com/go-chi/chi/middleware/closenotify17.go
+++ /dev/null
@@ -1,42 +0,0 @@
-// +build go1.7,!go1.8
-
-package middleware
-
-import (
-	"context"
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	fn := func(w http.ResponseWriter, r *http.Request) {
-		cn, ok := w.(http.CloseNotifier)
-		if !ok {
-			panic("chi/middleware: CloseNotify expects http.ResponseWriter to implement http.CloseNotifier interface")
-		}
-		closeNotifyCh := cn.CloseNotify()
-
-		ctx, cancel := context.WithCancel(r.Context())
-		defer cancel()
-
-		go func() {
-			select {
-			case <-ctx.Done():
-				return
-			case <-closeNotifyCh:
-				cancel()
-				return
-			}
-		}()
-
-		r = r.WithContext(ctx)
-		next.ServeHTTP(w, r)
-	}
-
-	return http.HandlerFunc(fn)
-}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go b/backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go
deleted file mode 100644
index 4f0d73cc..00000000
--- a/backend/vendor/github.com/go-chi/chi/middleware/closenotify18.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"net/http"
-)
-
-// CloseNotify is a middleware that cancels ctx when the underlying
-// connection has gone away. It can be used to cancel long operations
-// on the server when the client disconnects before the response is ready.
-//
-// Note: this behaviour is standard in Go 1.8+, so the middleware does nothing
-// on 1.8+ and exists just for backwards compatibility.
-func CloseNotify(next http.Handler) http.Handler {
-	return next
-}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/compress.go b/backend/vendor/github.com/go-chi/chi/middleware/compress.go
index 006ad48f..d2876d4e 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/compress.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/compress.go
@@ -11,24 +11,98 @@ import (
 	"strings"
 )
 
-type encoding int
+var encoders = map[string]EncoderFunc{}
 
-const (
-	encodingNone encoding = iota
-	encodingGzip
-	encodingDeflate
-)
+var encodingPrecedence = []string{"br", "gzip", "deflate"}
+
+func init() {
+	// TODO:
+	// lzma: Opera.
+	// sdch: Chrome, Android. Gzip output + dictionary header.
+	// br:   Brotli, see https://github.com/go-chi/chi/pull/326
+
+	// TODO: Exception for old MSIE browsers that can't handle non-HTML?
+	// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	SetEncoder("gzip", encoderGzip)
+
+	// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
+	// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
+	// checksum compared to CRC-32 used in "gzip" and thus is faster.
+	//
+	// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
+	// raw DEFLATE data only, without the mentioned zlib wrapper.
+	// Because of this major confusion, most modern browsers try it
+	// both ways, first looking for zlib headers.
+	// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
+	//
+	// The list of browsers having problems is quite big, see:
+	// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
+	// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
+	//
+	// That's why we prefer gzip over deflate. It's just more reliable
+	// and not significantly slower than gzip.
+	SetEncoder("deflate", encoderDeflate)
+
+	// NOTE: Not implemented, intentionally:
+	// case "compress": // LZW. Deprecated.
+	// case "bzip2":    // Too slow on-the-fly.
+	// case "zopfli":   // Too slow on-the-fly.
+	// case "xz":       // Too slow on-the-fly.
+}
+
+// An EncoderFunc is a function that wraps the provided ResponseWriter with a
+// streaming compression algorithm and returns it.
+//
+// In case of failure, the function should return nil.
+type EncoderFunc func(w http.ResponseWriter, level int) io.Writer
+
+// SetEncoder can be used to set the implementation of a compression algorithm.
+//
+// The encoding should be a standardised identifier. See:
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
+//
+// For example, add the Brotli algortithm:
+//
+//  import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc"
+//
+//  middleware.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer {
+//    params := brotli_enc.NewBrotliParams()
+//    params.SetQuality(level)
+//    return brotli_enc.NewBrotliWriter(params, w)
+//  })
+func SetEncoder(encoding string, fn EncoderFunc) {
+	encoding = strings.ToLower(encoding)
+	if encoding == "" {
+		panic("the encoding can not be empty")
+	}
+	if fn == nil {
+		panic("attempted to set a nil encoder function")
+	}
+	encoders[encoding] = fn
+
+	var e string
+	for _, v := range encodingPrecedence {
+		if v == encoding {
+			e = v
+		}
+	}
+
+	if e == "" {
+		encodingPrecedence = append([]string{e}, encodingPrecedence...)
+	}
+}
 
 var defaultContentTypes = map[string]struct{}{
-	"text/html":                struct{}{},
-	"text/css":                 struct{}{},
-	"text/plain":               struct{}{},
-	"text/javascript":          struct{}{},
-	"application/javascript":   struct{}{},
-	"application/x-javascript": struct{}{},
-	"application/json":         struct{}{},
-	"application/atom+xml":     struct{}{},
-	"application/rss+xml":      struct{}{},
+	"text/html":                {},
+	"text/css":                 {},
+	"text/plain":               {},
+	"text/javascript":          {},
+	"application/javascript":   {},
+	"application/x-javascript": {},
+	"application/json":         {},
+	"application/atom+xml":     {},
+	"application/rss+xml":      {},
+	"image/svg+xml":            {},
 }
 
 // DefaultCompress is a middleware that compresses response
@@ -43,6 +117,11 @@ func DefaultCompress(next http.Handler) http.Handler {
 // body of a given content types to a data format based
 // on Accept-Encoding request header. It uses a given
 // compression level.
+//
+// NOTE: make sure to set the Content-Type header on your response
+// otherwise this middleware will not compress the response body. For ex, in
+// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody))
+// or set it manually.
 func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 	contentTypes := defaultContentTypes
 	if len(types) > 0 {
@@ -54,159 +133,143 @@ func Compress(level int, types ...string) func(next http.Handler) http.Handler {
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
-			mcw := &maybeCompressResponseWriter{
+			encoder, encoding := selectEncoder(r.Header)
+
+			cw := &compressResponseWriter{
 				ResponseWriter: w,
 				w:              w,
 				contentTypes:   contentTypes,
-				encoding:       selectEncoding(r.Header),
+				encoder:        encoder,
+				encoding:       encoding,
 				level:          level,
 			}
-			defer mcw.Close()
+			defer cw.Close()
 
-			next.ServeHTTP(mcw, r)
+			next.ServeHTTP(cw, r)
 		}
 
 		return http.HandlerFunc(fn)
 	}
 }
 
-func selectEncoding(h http.Header) encoding {
-	enc := h.Get("Accept-Encoding")
+func selectEncoder(h http.Header) (EncoderFunc, string) {
+	header := h.Get("Accept-Encoding")
 
-	switch {
-	// TODO:
-	// case "br":    // Brotli, experimental. Firefox 2016, to-be-in Chromium.
-	// case "lzma":  // Opera.
-	// case "sdch":  // Chrome, Android. Gzip output + dictionary header.
+	// Parse the names of all accepted algorithms from the header.
+	accepted := strings.Split(strings.ToLower(header), ",")
 
-	case strings.Contains(enc, "gzip"):
-		// TODO: Exception for old MSIE browsers that can't handle non-HTML?
-		// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
-		return encodingGzip
-
-	case strings.Contains(enc, "deflate"):
-		// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
-		// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
-		// checksum compared to CRC-32 used in "gzip" and thus is faster.
-		//
-		// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
-		// raw DEFLATE data only, without the mentioned zlib wrapper.
-		// Because of this major confusion, most modern browsers try it
-		// both ways, first looking for zlib headers.
-		// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
-		//
-		// The list of browsers having problems is quite big, see:
-		// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
-		// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
-		//
-		// That's why we prefer gzip over deflate. It's just more reliable
-		// and not significantly slower than gzip.
-		return encodingDeflate
-
-		// NOTE: Not implemented, intentionally:
-		// case "compress": // LZW. Deprecated.
-		// case "bzip2":    // Too slow on-the-fly.
-		// case "zopfli":   // Too slow on-the-fly.
-		// case "xz":       // Too slow on-the-fly.
+	// Find supported encoder by accepted list by precedence
+	for _, name := range encodingPrecedence {
+		if fn, ok := encoders[name]; ok && matchAcceptEncoding(accepted, name) {
+			return fn, name
+		}
 	}
 
-	return encodingNone
+	// No encoder found to match the accepted encoding
+	return nil, ""
 }
 
-type maybeCompressResponseWriter struct {
+func matchAcceptEncoding(accepted []string, encoding string) bool {
+	for _, v := range accepted {
+		if strings.Index(v, encoding) >= 0 {
+			return true
+		}
+	}
+	return false
+}
+
+type compressResponseWriter struct {
 	http.ResponseWriter
 	w            io.Writer
-	encoding     encoding
+	encoder      EncoderFunc
+	encoding     string
 	contentTypes map[string]struct{}
 	level        int
 	wroteHeader  bool
 }
 
-func (w *maybeCompressResponseWriter) WriteHeader(code int) {
-	if w.wroteHeader {
+func (cw *compressResponseWriter) WriteHeader(code int) {
+	if cw.wroteHeader {
 		return
 	}
-	w.wroteHeader = true
-	defer w.ResponseWriter.WriteHeader(code)
+	cw.wroteHeader = true
+	defer cw.ResponseWriter.WriteHeader(code)
 
 	// Already compressed data?
-	if w.ResponseWriter.Header().Get("Content-Encoding") != "" {
+	if cw.Header().Get("Content-Encoding") != "" {
 		return
 	}
-	// The content-length after compression is unknown
-	w.ResponseWriter.Header().Del("Content-Length")
 
 	// Parse the first part of the Content-Type response header.
 	contentType := ""
-	parts := strings.Split(w.ResponseWriter.Header().Get("Content-Type"), ";")
+	parts := strings.Split(cw.Header().Get("Content-Type"), ";")
 	if len(parts) > 0 {
 		contentType = parts[0]
 	}
 
 	// Is the content type compressable?
-	if _, ok := w.contentTypes[contentType]; !ok {
+	if _, ok := cw.contentTypes[contentType]; !ok {
 		return
 	}
 
-	// Select the compress writer.
-	switch w.encoding {
-	case encodingGzip:
-		gw, err := gzip.NewWriterLevel(w.ResponseWriter, w.level)
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
-		}
-		w.w = gw
-		w.ResponseWriter.Header().Set("Content-Encoding", "gzip")
+	if cw.encoder != nil && cw.encoding != "" {
+		if wr := cw.encoder(cw.ResponseWriter, cw.level); wr != nil {
+			cw.w = wr
+			cw.Header().Set("Content-Encoding", cw.encoding)
 
-	case encodingDeflate:
-		dw, err := flate.NewWriter(w.ResponseWriter, w.level)
-		if err != nil {
-			w.w = w.ResponseWriter
-			return
+			// The content-length after compression is unknown
+			cw.Header().Del("Content-Length")
 		}
-		w.w = dw
-		w.ResponseWriter.Header().Set("Content-Encoding", "deflate")
 	}
 }
 
-func (w *maybeCompressResponseWriter) Write(p []byte) (int, error) {
-	if !w.wroteHeader {
-		w.WriteHeader(http.StatusOK)
+func (cw *compressResponseWriter) Write(p []byte) (int, error) {
+	if !cw.wroteHeader {
+		cw.WriteHeader(http.StatusOK)
 	}
 
-	return w.w.Write(p)
+	return cw.w.Write(p)
 }
 
-func (w *maybeCompressResponseWriter) Flush() {
-	if f, ok := w.w.(http.Flusher); ok {
+func (cw *compressResponseWriter) Flush() {
+	if f, ok := cw.w.(http.Flusher); ok {
 		f.Flush()
 	}
 }
 
-func (w *maybeCompressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	if hj, ok := w.w.(http.Hijacker); ok {
+func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	if hj, ok := cw.w.(http.Hijacker); ok {
 		return hj.Hijack()
 	}
 	return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer")
 }
 
-func (w *maybeCompressResponseWriter) CloseNotify() <-chan bool {
-	if cn, ok := w.w.(http.CloseNotifier); ok {
-		return cn.CloseNotify()
+func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
+	if ps, ok := cw.w.(http.Pusher); ok {
+		return ps.Push(target, opts)
 	}
-
-	// If the underlying writer does not implement http.CloseNotifier, return
-	// a channel that never receives a value. The semantics here is that the
-	// client never disconnnects before the request is processed by the
-	// http.Handler, which is close enough to the default behavior (when
-	// CloseNotify() is not even called).
-	return make(chan bool, 1)
+	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
 }
 
-func (w *maybeCompressResponseWriter) Close() error {
-	if c, ok := w.w.(io.WriteCloser); ok {
+func (cw *compressResponseWriter) Close() error {
+	if c, ok := cw.w.(io.WriteCloser); ok {
 		return c.Close()
 	}
 	return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer")
 }
+
+func encoderGzip(w http.ResponseWriter, level int) io.Writer {
+	gw, err := gzip.NewWriterLevel(w, level)
+	if err != nil {
+		return nil
+	}
+	return gw
+}
+
+func encoderDeflate(w http.ResponseWriter, level int) io.Writer {
+	dw, err := flate.NewWriter(w, level)
+	if err != nil {
+		return nil
+	}
+	return dw
+}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/compress18.go b/backend/vendor/github.com/go-chi/chi/middleware/compress18.go
deleted file mode 100644
index 0048f7d9..00000000
--- a/backend/vendor/github.com/go-chi/chi/middleware/compress18.go
+++ /dev/null
@@ -1,15 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"errors"
-	"net/http"
-)
-
-func (w *maybeCompressResponseWriter) Push(target string, opts *http.PushOptions) error {
-	if ps, ok := w.w.(http.Pusher); ok {
-		return ps.Push(target, opts)
-	}
-	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
-}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/content_type.go b/backend/vendor/github.com/go-chi/chi/middleware/content_type.go
index 3a2dc20a..ee495787 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/content_type.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/content_type.go
@@ -26,6 +26,12 @@ func AllowContentType(contentTypes ...string) func(next http.Handler) http.Handl
 
 	return func(next http.Handler) http.Handler {
 		fn := func(w http.ResponseWriter, r *http.Request) {
+			if r.ContentLength == 0 {
+				// skip check for empty content body
+				next.ServeHTTP(w, r)
+				return
+			}
+
 			s := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Type")))
 			if i := strings.Index(s, ";"); i > -1 {
 				s = s[0:i]
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/logger.go b/backend/vendor/github.com/go-chi/chi/middleware/logger.go
index 99fac03d..9f119d56 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/logger.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/logger.go
@@ -16,7 +16,7 @@ var (
 	// DefaultLogger is called by the Logger middleware handler to log each request.
 	// Its made a package-level variable so that it can be reconfigured for custom
 	// logging configurations.
-	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags)})
+	DefaultLogger = RequestLogger(&DefaultLogFormatter{Logger: log.New(os.Stdout, "", log.LstdFlags), NoColor: false})
 )
 
 // Logger is a middleware that logs the start and end of each request, along
@@ -81,29 +81,32 @@ type LoggerInterface interface {
 
 // DefaultLogFormatter is a simple logger that implements a LogFormatter.
 type DefaultLogFormatter struct {
-	Logger LoggerInterface
+	Logger  LoggerInterface
+	NoColor bool
 }
 
 // NewLogEntry creates a new LogEntry for the request.
 func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
+	useColor := !l.NoColor
 	entry := &defaultLogEntry{
 		DefaultLogFormatter: l,
 		request:             r,
 		buf:                 &bytes.Buffer{},
+		useColor:            useColor,
 	}
 
 	reqID := GetReqID(r.Context())
 	if reqID != "" {
-		cW(entry.buf, nYellow, "[%s] ", reqID)
+		cW(entry.buf, useColor, nYellow, "[%s] ", reqID)
 	}
-	cW(entry.buf, nCyan, "\"")
-	cW(entry.buf, bMagenta, "%s ", r.Method)
+	cW(entry.buf, useColor, nCyan, "\"")
+	cW(entry.buf, useColor, bMagenta, "%s ", r.Method)
 
 	scheme := "http"
 	if r.TLS != nil {
 		scheme = "https"
 	}
-	cW(entry.buf, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
+	cW(entry.buf, useColor, nCyan, "%s://%s%s %s\" ", scheme, r.Host, r.RequestURI, r.Proto)
 
 	entry.buf.WriteString("from ")
 	entry.buf.WriteString(r.RemoteAddr)
@@ -114,33 +117,34 @@ func (l *DefaultLogFormatter) NewLogEntry(r *http.Request) LogEntry {
 
 type defaultLogEntry struct {
 	*DefaultLogFormatter
-	request *http.Request
-	buf     *bytes.Buffer
+	request  *http.Request
+	buf      *bytes.Buffer
+	useColor bool
 }
 
 func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 	switch {
 	case status < 200:
-		cW(l.buf, bBlue, "%03d", status)
+		cW(l.buf, l.useColor, bBlue, "%03d", status)
 	case status < 300:
-		cW(l.buf, bGreen, "%03d", status)
+		cW(l.buf, l.useColor, bGreen, "%03d", status)
 	case status < 400:
-		cW(l.buf, bCyan, "%03d", status)
+		cW(l.buf, l.useColor, bCyan, "%03d", status)
 	case status < 500:
-		cW(l.buf, bYellow, "%03d", status)
+		cW(l.buf, l.useColor, bYellow, "%03d", status)
 	default:
-		cW(l.buf, bRed, "%03d", status)
+		cW(l.buf, l.useColor, bRed, "%03d", status)
 	}
 
-	cW(l.buf, bBlue, " %dB", bytes)
+	cW(l.buf, l.useColor, bBlue, " %dB", bytes)
 
 	l.buf.WriteString(" in ")
 	if elapsed < 500*time.Millisecond {
-		cW(l.buf, nGreen, "%s", elapsed)
+		cW(l.buf, l.useColor, nGreen, "%s", elapsed)
 	} else if elapsed < 5*time.Second {
-		cW(l.buf, nYellow, "%s", elapsed)
+		cW(l.buf, l.useColor, nYellow, "%s", elapsed)
 	} else {
-		cW(l.buf, nRed, "%s", elapsed)
+		cW(l.buf, l.useColor, nRed, "%s", elapsed)
 	}
 
 	l.Logger.Print(l.buf.String())
@@ -148,7 +152,7 @@ func (l *defaultLogEntry) Write(status, bytes int, elapsed time.Duration) {
 
 func (l *defaultLogEntry) Panic(v interface{}, stack []byte) {
 	panicEntry := l.NewLogEntry(l.request).(*defaultLogEntry)
-	cW(panicEntry.buf, bRed, "panic: %+v", v)
+	cW(panicEntry.buf, l.useColor, bRed, "panic: %+v", v)
 	l.Logger.Print(panicEntry.buf.String())
 	l.Logger.Print(string(stack))
 }
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/nocache.go b/backend/vendor/github.com/go-chi/chi/middleware/nocache.go
index e5819ddd..2412829e 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/nocache.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/nocache.go
@@ -14,7 +14,7 @@ var epoch = time.Unix(0, 0).Format(time.RFC1123)
 // Taken from https://github.com/mytrile/nocache
 var noCacheHeaders = map[string]string{
 	"Expires":         epoch,
-	"Cache-Control":   "no-cache, no-store, must-revalidate, private, max-age=0",
+	"Cache-Control":   "no-cache, no-store, no-transform, must-revalidate, private, max-age=0",
 	"Pragma":          "no-cache",
 	"X-Accel-Expires": "0",
 }
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/realip.go b/backend/vendor/github.com/go-chi/chi/middleware/realip.go
index e9addbe3..146c2b0a 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/realip.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/realip.go
@@ -22,7 +22,7 @@ var xRealIP = http.CanonicalHeaderKey("X-Real-IP")
 // You should only use this middleware if you can trust the headers passed to
 // you (in particular, the two headers this middleware uses), for example
 // because you have placed a reverse proxy like HAProxy or nginx in front of
-// Goji. If your reverse proxies are configured to pass along arbitrary header
+// chi. If your reverse proxies are configured to pass along arbitrary header
 // values from the client, or if you use this middleware without a reverse
 // proxy, malicious clients will be able to make you very sad (or, depending on
 // how you're using RemoteAddr, vulnerable to an attack of some sort).
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/request_id.go b/backend/vendor/github.com/go-chi/chi/middleware/request_id.go
index 4574bde8..65b58f63 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/request_id.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/request_id.go
@@ -17,7 +17,7 @@ import (
 // Key to use when setting the request ID.
 type ctxKeyRequestID int
 
-// RequestIDKey is the key that holds th unique request ID in a request context.
+// RequestIDKey is the key that holds the unique request ID in a request context.
 const RequestIDKey ctxKeyRequestID = 0
 
 var prefix string
@@ -62,9 +62,13 @@ func init() {
 // counter.
 func RequestID(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
-		myid := atomic.AddUint64(&reqid, 1)
 		ctx := r.Context()
-		ctx = context.WithValue(ctx, RequestIDKey, fmt.Sprintf("%s-%06d", prefix, myid))
+		requestID := r.Header.Get("X-Request-Id")
+		if requestID == "" {
+			myid := atomic.AddUint64(&reqid, 1)
+			requestID = fmt.Sprintf("%s-%06d", prefix, myid)
+		}
+		ctx = context.WithValue(ctx, RequestIDKey, requestID)
 		next.ServeHTTP(w, r.WithContext(ctx))
 	}
 	return http.HandlerFunc(fn)
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/strip.go b/backend/vendor/github.com/go-chi/chi/middleware/strip.go
index 8f19766b..2b8b1842 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/strip.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/strip.go
@@ -1,6 +1,7 @@
 package middleware
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/go-chi/chi"
@@ -28,6 +29,9 @@ func StripSlashes(next http.Handler) http.Handler {
 
 // RedirectSlashes is a middleware that will match request paths with a trailing
 // slash and redirect to the same path, less the trailing slash.
+//
+// NOTE: RedirectSlashes middleware is *incompatible* with http.FileServer,
+// see https://github.com/go-chi/chi/issues/343
 func RedirectSlashes(next http.Handler) http.Handler {
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		var path string
@@ -38,7 +42,11 @@ func RedirectSlashes(next http.Handler) http.Handler {
 			path = r.URL.Path
 		}
 		if len(path) > 1 && path[len(path)-1] == '/' {
-			path = path[:len(path)-1]
+			if r.URL.RawQuery != "" {
+				path = fmt.Sprintf("%s?%s", path[:len(path)-1], r.URL.RawQuery)
+			} else {
+				path = path[:len(path)-1]
+			}
 			http.Redirect(w, r, path, 301)
 			return
 		}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/terminal.go b/backend/vendor/github.com/go-chi/chi/middleware/terminal.go
index 79930a25..a5d42410 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/terminal.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/terminal.go
@@ -52,12 +52,12 @@ func init() {
 }
 
 // colorWrite
-func cW(w io.Writer, color []byte, s string, args ...interface{}) {
-	if isTTY {
+func cW(w io.Writer, useColor bool, color []byte, s string, args ...interface{}) {
+	if isTTY && useColor {
 		w.Write(color)
 	}
 	fmt.Fprintf(w, s, args...)
-	if isTTY {
+	if isTTY && useColor {
 		w.Write(reset)
 	}
 }
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/timeout.go b/backend/vendor/github.com/go-chi/chi/middleware/timeout.go
index 5cabf1f9..8e373536 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/timeout.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/timeout.go
@@ -15,7 +15,8 @@ import (
 //
 // ie. a route/handler may look like:
 //
-//  r.Get("/long", func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+//  r.Get("/long", func(w http.ResponseWriter, r *http.Request) {
+// 	 ctx := r.Context()
 // 	 processTime := time.Duration(rand.Intn(4)+1) * time.Second
 //
 // 	 select {
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer.go b/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer.go
index 5d1c286b..5e5594f8 100644
--- a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer.go
+++ b/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer.go
@@ -10,6 +10,32 @@ import (
 	"net/http"
 )
 
+// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
+// hook into various parts of the response process.
+func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
+	_, fl := w.(http.Flusher)
+
+	bw := basicWriter{ResponseWriter: w}
+
+	if protoMajor == 2 {
+		_, ps := w.(http.Pusher)
+		if fl && ps {
+			return &http2FancyWriter{bw}
+		}
+	} else {
+		_, hj := w.(http.Hijacker)
+		_, rf := w.(io.ReaderFrom)
+		if fl && hj && rf {
+			return &httpFancyWriter{bw}
+		}
+	}
+	if fl {
+		return &flushWriter{bw}
+	}
+
+	return &bw
+}
+
 // WrapResponseWriter is a proxy around an http.ResponseWriter that allows you to hook
 // into various parts of the response process.
 type WrapResponseWriter interface {
@@ -47,6 +73,7 @@ func (b *basicWriter) WriteHeader(code int) {
 		b.ResponseWriter.WriteHeader(code)
 	}
 }
+
 func (b *basicWriter) Write(buf []byte) (int, error) {
 	b.WriteHeader(http.StatusOK)
 	n, err := b.ResponseWriter.Write(buf)
@@ -60,20 +87,25 @@ func (b *basicWriter) Write(buf []byte) (int, error) {
 	b.bytes += n
 	return n, err
 }
+
 func (b *basicWriter) maybeWriteHeader() {
 	if !b.wroteHeader {
 		b.WriteHeader(http.StatusOK)
 	}
 }
+
 func (b *basicWriter) Status() int {
 	return b.code
 }
+
 func (b *basicWriter) BytesWritten() int {
 	return b.bytes
 }
+
 func (b *basicWriter) Tee(w io.Writer) {
 	b.tee = w
 }
+
 func (b *basicWriter) Unwrap() http.ResponseWriter {
 	return b.ResponseWriter
 }
@@ -83,13 +115,15 @@ type flushWriter struct {
 }
 
 func (f *flushWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
 
 var _ http.Flusher = &flushWriter{}
 
-// httpFancyWriter is a HTTP writer that additionally satisfies http.CloseNotifier,
+// httpFancyWriter is a HTTP writer that additionally satisfies
 // http.Flusher, http.Hijacker, and io.ReaderFrom. It exists for the common case
 // of wrapping the http.ResponseWriter that package http gives you, in order to
 // make the proxied object support the full method set of the proxied object.
@@ -97,18 +131,22 @@ type httpFancyWriter struct {
 	basicWriter
 }
 
-func (f *httpFancyWriter) CloseNotify() <-chan bool {
-	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
-	return cn.CloseNotify()
-}
 func (f *httpFancyWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
+
 func (f *httpFancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 	hj := f.basicWriter.ResponseWriter.(http.Hijacker)
 	return hj.Hijack()
 }
+
+func (f *http2FancyWriter) Push(target string, opts *http.PushOptions) error {
+	return f.basicWriter.ResponseWriter.(http.Pusher).Push(target, opts)
+}
+
 func (f *httpFancyWriter) ReadFrom(r io.Reader) (int64, error) {
 	if f.basicWriter.tee != nil {
 		n, err := io.Copy(&f.basicWriter, r)
@@ -122,12 +160,12 @@ func (f *httpFancyWriter) ReadFrom(r io.Reader) (int64, error) {
 	return n, err
 }
 
-var _ http.CloseNotifier = &httpFancyWriter{}
 var _ http.Flusher = &httpFancyWriter{}
 var _ http.Hijacker = &httpFancyWriter{}
+var _ http.Pusher = &http2FancyWriter{}
 var _ io.ReaderFrom = &httpFancyWriter{}
 
-// http2FancyWriter is a HTTP2 writer that additionally satisfies http.CloseNotifier,
+// http2FancyWriter is a HTTP2 writer that additionally satisfies
 // http.Flusher, and io.ReaderFrom. It exists for the common case
 // of wrapping the http.ResponseWriter that package http gives you, in order to
 // make the proxied object support the full method set of the proxied object.
@@ -135,14 +173,11 @@ type http2FancyWriter struct {
 	basicWriter
 }
 
-func (f *http2FancyWriter) CloseNotify() <-chan bool {
-	cn := f.basicWriter.ResponseWriter.(http.CloseNotifier)
-	return cn.CloseNotify()
-}
 func (f *http2FancyWriter) Flush() {
+	f.wroteHeader = true
+
 	fl := f.basicWriter.ResponseWriter.(http.Flusher)
 	fl.Flush()
 }
 
-var _ http.CloseNotifier = &http2FancyWriter{}
 var _ http.Flusher = &http2FancyWriter{}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer17.go b/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer17.go
deleted file mode 100644
index c60df608..00000000
--- a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer17.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// +build go1.7,!go1.8
-
-package middleware
-
-import (
-	"io"
-	"net/http"
-)
-
-// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
-// hook into various parts of the response process.
-func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
-	_, cn := w.(http.CloseNotifier)
-	_, fl := w.(http.Flusher)
-
-	bw := basicWriter{ResponseWriter: w}
-
-	if protoMajor == 2 {
-		if cn && fl {
-			return &http2FancyWriter{bw}
-		}
-	} else {
-		_, hj := w.(http.Hijacker)
-		_, rf := w.(io.ReaderFrom)
-		if cn && fl && hj && rf {
-			return &httpFancyWriter{bw}
-		}
-	}
-	if fl {
-		return &flushWriter{bw}
-	}
-
-	return &bw
-}
diff --git a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer18.go b/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer18.go
deleted file mode 100644
index 115c2d4f..00000000
--- a/backend/vendor/github.com/go-chi/chi/middleware/wrap_writer18.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// +build go1.8 appengine
-
-package middleware
-
-import (
-	"io"
-	"net/http"
-)
-
-// NewWrapResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
-// hook into various parts of the response process.
-func NewWrapResponseWriter(w http.ResponseWriter, protoMajor int) WrapResponseWriter {
-	_, cn := w.(http.CloseNotifier)
-	_, fl := w.(http.Flusher)
-
-	bw := basicWriter{ResponseWriter: w}
-
-	if protoMajor == 2 {
-		_, ps := w.(http.Pusher)
-		if cn && fl && ps {
-			return &http2FancyWriter{bw}
-		}
-	} else {
-		_, hj := w.(http.Hijacker)
-		_, rf := w.(io.ReaderFrom)
-		if cn && fl && hj && rf {
-			return &httpFancyWriter{bw}
-		}
-	}
-	if fl {
-		return &flushWriter{bw}
-	}
-
-	return &bw
-}
-
-func (f *http2FancyWriter) Push(target string, opts *http.PushOptions) error {
-	return f.basicWriter.ResponseWriter.(http.Pusher).Push(target, opts)
-}
-
-var _ http.Pusher = &http2FancyWriter{}
diff --git a/backend/vendor/github.com/go-chi/chi/mux.go b/backend/vendor/github.com/go-chi/chi/mux.go
index 84a2424a..e553287e 100644
--- a/backend/vendor/github.com/go-chi/chi/mux.go
+++ b/backend/vendor/github.com/go-chi/chi/mux.go
@@ -60,7 +60,8 @@ func NewMux() *Mux {
 func (mx *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Ensure the mux has some routes defined on the mux
 	if mx.handler == nil {
-		panic("chi: attempting to route to a mux with no handlers.")
+		mx.NotFoundHandler().ServeHTTP(w, r)
+		return
 	}
 
 	// Check if a routing context already exists from a parent router.
diff --git a/backend/vendor/github.com/go-chi/chi/tree.go b/backend/vendor/github.com/go-chi/chi/tree.go
index a55d7f14..8a044f3e 100644
--- a/backend/vendor/github.com/go-chi/chi/tree.go
+++ b/backend/vendor/github.com/go-chi/chi/tree.go
@@ -33,15 +33,15 @@ var mALL = mCONNECT | mDELETE | mGET | mHEAD |
 	mOPTIONS | mPATCH | mPOST | mPUT | mTRACE
 
 var methodMap = map[string]methodTyp{
-	"CONNECT": mCONNECT,
-	"DELETE":  mDELETE,
-	"GET":     mGET,
-	"HEAD":    mHEAD,
-	"OPTIONS": mOPTIONS,
-	"PATCH":   mPATCH,
-	"POST":    mPOST,
-	"PUT":     mPUT,
-	"TRACE":   mTRACE,
+	http.MethodConnect: mCONNECT,
+	http.MethodDelete:  mDELETE,
+	http.MethodGet:     mGET,
+	http.MethodHead:    mHEAD,
+	http.MethodOptions: mOPTIONS,
+	http.MethodPatch:   mPATCH,
+	http.MethodPost:    mPOST,
+	http.MethodPut:     mPUT,
+	http.MethodTrace:   mTRACE,
 }
 
 // RegisterMethod adds support for custom HTTP method handlers, available
@@ -706,7 +706,9 @@ func patNextSegment(pattern string) (nodeTyp, string, string, byte, int, int) {
 	}
 
 	// Wildcard pattern as finale
-	// TODO: should we panic if there is stuff after the * ???
+	if ws < len(pattern)-1 {
+		panic("chi: wildcard '*' must be the last value in a route. trim trailing text or use a '{param}' instead")
+	}
 	return ntCatchAll, "*", "", 0, ws, len(pattern)
 }
 
diff --git a/backend/vendor/github.com/go-pkgz/lgr/.golangci.yml b/backend/vendor/github.com/go-pkgz/lgr/.golangci.yml
new file mode 100644
index 00000000..989af86e
--- /dev/null
+++ b/backend/vendor/github.com/go-pkgz/lgr/.golangci.yml
@@ -0,0 +1,60 @@
+linters-settings:
+  govet:
+    check-shadowing: true
+  golint:
+    min-confidence: 0
+  gocyclo:
+    min-complexity: 15
+  maligned:
+    suggest-new: true
+  dupl:
+    threshold: 100
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+
+linters:
+  disable-all: true
+  enable:
+    - megacheck
+    - govet
+    - unconvert
+    - megacheck
+    - structcheck
+    - gas
+    - gocyclo
+    - dupl
+    - misspell
+    - unparam
+    - varcheck
+    - deadcode
+    - typecheck
+    - ineffassign
+    - varcheck
+  fast: false
+
+
+run:
+#  modules-download-mode: vendor
+  skip-dirs:
+    - vendor
+
+issues:
+  exclude-rules:
+    - text: "weak cryptographic primitive"
+      linters:
+        - gosec
+
+service:
+  golangci-lint-version: 1.16.x
\ No newline at end of file
diff --git a/backend/vendor/github.com/go-pkgz/lgr/.travis.yml b/backend/vendor/github.com/go-pkgz/lgr/.travis.yml
index 4e4d2ade..189a75c8 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/.travis.yml
+++ b/backend/vendor/github.com/go-pkgz/lgr/.travis.yml
@@ -13,7 +13,6 @@ before_install:
 
 script:
   - GO111MODULE=on go get ./...
-  - GO111MODULE=on go mod vendor
-  - GO111MODULE=on go test  -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
-  - golangci-lint run  || travis_terminate 1;
+  - GO111MODULE=on go test -v -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
+  - golangci-lint run || travis_terminate 1;
   - $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci
diff --git a/backend/vendor/github.com/go-pkgz/lgr/README.md b/backend/vendor/github.com/go-pkgz/lgr/README.md
index 77d81424..33559091 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/README.md
+++ b/backend/vendor/github.com/go-pkgz/lgr/README.md
@@ -32,7 +32,7 @@ _Without `lgr.Caller*` it will drop `{caller}` part_
 
 `lgr.New` call accepts functional options:
 
-- `lgr.Debug` - turn debug mode on to allow messages with "DEBUG" level (filtered overwise)
+- `lgr.Debug` - turn debug mode on to allow messages with "DEBUG" level (filtered otherwise)
 - `lgr.Out(io.Writer)` - sets the output writer, default `os.Stdout`
 - `lgr.Err(io.Writer)` - sets the error writer, default `os.Stderr`
 - `lgr.CallerFile` - adds the caller file info
@@ -42,6 +42,8 @@ _Without `lgr.Caller*` it will drop `{caller}` part_
 - `lgr.Msec` - adds milliseconds to timestamp
 - `lgr.Format` - sets custom template, overwrite all other formatting modifiers.
 
+example: `l := lgr.New(lgr.Debug, lgr.Msec)`
+
 #### formatting templates:
 
 Several predefined templates provided and can be passed directly to `lgr.Format`, i.e. `lgr.Format(lgr.WithMsec)`
@@ -58,9 +60,11 @@ Several predefined templates provided and can be passed directly to `lgr.Format`
 User can make a custom template and pass it directly to `lgr.Format`. For example:
 
 ```go    
-    lgr.Format(`{{.Level}} - {{.DT.Format "2006-01-02T15:04:05Z07:00") - {{.CallerPkg}} - {{.Message}}`)
+    lgr.Format(`{{.Level}} - {{.DT.Format "2006-01-02T15:04:05Z07:00"}} - {{.CallerPkg}} - {{.Message}}`)
 ```
-)
+
+_Note: formatter (predefined or custom) adds measurable overhead - the cost will depend on the version of Go, but is between 30
+ and 50% in recent tests with 1.12. You can validate this in your environment via benchmarks: `go test -bench=. -run=Bench`_
     
 ### levels
 
@@ -70,8 +74,9 @@ User can make a custom template and pass it directly to `lgr.Format`. For exampl
 - `DEBUG` will be filtered unless `lgr.Debug` or `lgr.Trace` options defined
 - `INFO` and `WARN` don't have any special behavior attached
 - `ERROR` sends messages to both out and err writers
-- `PANIC` and `FATAL` send messages to both out and err writers. In addition sends dump of callers and runtime info to err only, and calls `os.Exit(1)`.
-
+- `FATAL` and send messages to both out and err writers and exit(1)
+- `PANIC` does the same as `FATAL` but in addition sends dump of callers and runtime info to err.
+ 
 ### adaptors
 
 `lgr` logger can be converted to `io.Writer` or `*log.Logger`
@@ -79,11 +84,10 @@ User can make a custom template and pass it directly to `lgr.Format`. For exampl
 - `lgr.ToWriter(l lgr.L, level string) io.Writer` - makes io.Writer forwarding write ops to underlying `lgr.L`
 - `lgr.ToStdLogger(l lgr.L, level string) *log.Logger` - makes standard logger on top of `lgr.L`
 
-_`level` parameter is optional, if defined will enforce the level._    
+_`level` parameter is optional, if defined (non-empty) will enforce the level._    
   
 ### global logger
 
 Users **should avoid** global logger and pass the concrete logger as a dependency. However, in some cases a global logger may be needed, for example migration from stdlib `log` to `lgr`. For such cases `log "github.com/go-pkgz/lgr"` can be imported instead of `log` package.
 
 Global logger provides `lgr.Printf`, `lgr.Print` and `lgr.Fatalf` functions. User can customize the logger by calling `lgr.Setup(options ...)`. The instance of this logger can be retrieved with `lgr.Default()`
- 
diff --git a/backend/vendor/github.com/go-pkgz/lgr/adaptor.go b/backend/vendor/github.com/go-pkgz/lgr/adaptor.go
index 3c213c2f..2627325d 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/adaptor.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/adaptor.go
@@ -11,9 +11,9 @@ type Writer struct {
 	level string // if defined added to each message
 }
 
-// Write to lgr.L, trim EOL
+// Write to lgr.L
 func (w *Writer) Write(p []byte) (n int, err error) {
-	w.Logf(strings.TrimSuffix(w.level+string(p), "\n"))
+	w.Logf(w.level + string(p))
 	return len(p), nil
 }
 
diff --git a/backend/vendor/github.com/go-pkgz/lgr/interface.go b/backend/vendor/github.com/go-pkgz/lgr/interface.go
index dbe1969a..46b961aa 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/interface.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/interface.go
@@ -2,7 +2,6 @@ package lgr
 
 import (
 	stdlog "log"
-	"os"
 )
 
 var def = New() // default logger doesn't allow DEBUG and doesn't add caller info
@@ -37,7 +36,7 @@ func Print(line string) {
 // Fatalf simplifies replacement of std logger
 func Fatalf(format string, args ...interface{}) {
 	def.logf(format, args...)
-	os.Exit(1)
+	def.fatal()
 }
 
 // Setup default logger with options
diff --git a/backend/vendor/github.com/go-pkgz/lgr/logger.go b/backend/vendor/github.com/go-pkgz/lgr/logger.go
index 264c9b2e..2dd86885 100644
--- a/backend/vendor/github.com/go-pkgz/lgr/logger.go
+++ b/backend/vendor/github.com/go-pkgz/lgr/logger.go
@@ -1,10 +1,11 @@
 // Package lgr provides a simple logger with some extras. Primary way to log is Logf method.
 // The logger's output can be customized in 2 ways:
-//   - by passing formatting template, i.e. lgr.New(lgr.Format(lgr.Short))
 //   - by setting individual formatting flags, i.e. lgr.New(lgr.Msec, lgr.CallerFunc)
-// Leveled output works for messages based on level prefix, i.e. Logf("INFO some message") means INFO level.
+//   - by passing formatting template, i.e. lgr.New(lgr.Format(lgr.Short))
+// Leveled output works for messages based on text prefix, i.e. Logf("INFO some message") means INFO level.
 // Debug and trace levels can be filtered based on lgr.Trace and lgr.Debug options.
-// ERROR, FATAL and PANIC levels send to err as well. Both FATAL and PANIC also print stack trace and terminate caller application with os.Exit(1)
+// ERROR, FATAL and PANIC levels send to err as well. FATAL terminate caller application with os.Exit(1)
+// and PANIC also prints stack trace.
 
 package lgr
 
@@ -15,6 +16,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"text/template"
@@ -59,7 +61,7 @@ type Logger struct {
 type nowFn func() time.Time
 type panicFn func()
 
-// layout holds all parts to construct the final message with template
+// layout holds all parts to construct the final message with template or with individual flags
 type layout struct {
 	DT         time.Time
 	Level      string
@@ -85,27 +87,28 @@ func New(options ...Option) *Logger {
 		opt(&res)
 	}
 
-	var err error
-	if res.format == "" {
-		res.format = res.templateFromOptions()
+	if res.format != "" {
+		// formatter defined
+		var err error
+		res.templ, err = template.New("lgr").Parse(res.format)
+		if err != nil {
+			fmt.Printf("invalid template %s, error %v. switched to %s\n", res.format, err, Short)
+			res.format = Short
+			res.templ = template.Must(template.New("lgrDefault").Parse(Short))
+		}
+
+		buf := bytes.Buffer{}
+		if err = res.templ.Execute(&buf, layout{}); err != nil {
+			fmt.Printf("failed to execute template %s, error %v. switched to %s\n", res.format, err, Short)
+			res.format = Short
+			res.templ = template.Must(template.New("lgrDefault").Parse(Short))
+		}
 	}
 
-	res.templ, err = template.New("lgr").Parse(res.format)
-	if err != nil {
-		fmt.Printf("invalid template %s, error %v. switched to %s\n", res.format, err, Short)
-		res.format = Short
-		res.templ = template.Must(template.New("lgrDefault").Parse(Short))
-	}
+	// set *On flags once for optimization on multiple Logf calls
+	res.callerOn = strings.Contains(res.format, "{{.Caller") || res.callerFile || res.callerFunc || res.callerPkg
+	res.levelBracesOn = strings.Contains(res.format, "[{{.Level}}]") || res.levelBraces
 
-	buf := bytes.Buffer{}
-	if err = res.templ.Execute(&buf, layout{}); err != nil {
-		fmt.Printf("failed to execute template %s, error %v. switched to %s\n", res.format, err, Short)
-		res.format = Short
-		res.templ = template.Must(template.New("lgrDefault").Parse(Short))
-	}
-
-	res.callerOn = strings.Contains(res.format, "{{.Caller")
-	res.levelBracesOn = strings.Contains(res.format, "[{{.Level}}]")
 	return &res
 }
 
@@ -128,7 +131,7 @@ func (l *Logger) logf(format string, args ...interface{}) {
 		return
 	}
 
-	ci := callerInfo{}
+	var ci callerInfo
 	if l.callerOn { // optimization to avoid expensive caller evaluation if caller info not in the template
 		ci = l.reportCaller(l.callerDepth)
 	}
@@ -136,21 +139,26 @@ func (l *Logger) logf(format string, args ...interface{}) {
 	elems := layout{
 		DT:         l.now(),
 		Level:      l.formatLevel(lv),
-		Message:    strings.TrimSuffix(msg, "\n"),
+		Message:    strings.TrimSuffix(msg, "\n"), // output adds EOL, trim from the message if passed
 		CallerFunc: ci.FuncName,
 		CallerFile: ci.File,
 		CallerPkg:  ci.Pkg,
 		CallerLine: ci.Line,
 	}
 
-	buf := bytes.Buffer{}
-	err := l.templ.Execute(&buf, elems) // once constructed, a template may be executed safely in parallel.
-	if err != nil {
-		fmt.Printf("failed to execute template, %v\n", err)
+	var data []byte
+	if l.format == "" {
+		data = []byte(l.formatWithOptions(elems))
+	} else {
+		buf := bytes.Buffer{}
+		err := l.templ.Execute(&buf, elems) // once constructed, a template may be executed safely in parallel.
+		if err != nil {
+			fmt.Printf("failed to execute template, %v\n", err) // should never happen
+		}
+		data = buf.Bytes()
 	}
-	buf.WriteString("\n")
+	data = append(data, '\n')
 
-	data := buf.Bytes()
 	if l.levelBracesOn { // rearrange space in short levels
 		data = bytes.Replace(data, []byte("[WARN ]"), []byte("[WARN] "), 1)
 		data = bytes.Replace(data, []byte("[INFO ]"), []byte("[INFO] "), 1)
@@ -230,51 +238,51 @@ func (l *Logger) reportCaller(calldepth int) (res callerInfo) {
 	return res
 }
 
-// make template from option flags
-func (l *Logger) templateFromOptions() (res string) {
+// speed-optimized version of formatter, used with individual options only, i.e. without Format call
+func (l *Logger) formatWithOptions(elems layout) (res string) {
 
-	const (
-		// escape { and } from templates to allow "{some/blah}" output for caller
-		openCallerBrace  = `{{"{"}}`
-		closeCallerBrace = `{{"}"}}`
-	)
-
-	orElse := func(flag bool, value string, elseValue string) string {
+	orElse := func(flag bool, fnTrue func() string, fnFalse func() string) string {
 		if flag {
-			return value
+			return fnTrue()
 		}
-		return elseValue
+		return fnFalse()
 	}
+	nothing := func() string { return "" }
 
-	var parts []string
+	parts := make([]string, 0, 4)
 
-	parts = append(parts, orElse(l.msec, `{{.DT.Format "2006/01/02 15:04:05.000"}}`, `{{.DT.Format "2006/01/02 15:04:05"}}`))
-	parts = append(parts, orElse(l.levelBraces, `[{{.Level}}]`, `{{.Level}}`))
+	parts = append(parts, orElse(l.msec,
+		func() string { return elems.DT.Format("2006/01/02 15:04:05.000") },
+		func() string { return elems.DT.Format("2006/01/02 15:04:05") },
+	))
+
+	parts = append(parts, orElse(l.levelBraces,
+		func() string { return `[` + elems.Level + `]` },
+		func() string { return elems.Level },
+	))
 
 	if l.callerFile || l.callerFunc || l.callerPkg {
 		var callerParts []string
-		if v := orElse(l.callerFile, `{{.CallerFile}}:{{.CallerLine}}`, ""); v != "" {
+		v := orElse(l.callerFile, func() string { return elems.CallerFile + ":" + strconv.Itoa(elems.CallerLine) }, nothing)
+		if v != "" {
 			callerParts = append(callerParts, v)
 		}
-		if v := orElse(l.callerFunc, `{{.CallerFunc}}`, ""); v != "" {
+		if v := orElse(l.callerFunc, func() string { return elems.CallerFunc }, nothing); v != "" {
 			callerParts = append(callerParts, v)
 		}
-		if v := orElse(l.callerPkg, `{{.CallerPkg}}`, ""); v != "" {
+		if v := orElse(l.callerPkg, func() string { return elems.CallerPkg }, nothing); v != "" {
 			callerParts = append(callerParts, v)
 		}
-		parts = append(parts, openCallerBrace+strings.Join(callerParts, " ")+closeCallerBrace)
+		parts = append(parts, "{"+strings.Join(callerParts, " ")+"}")
 	}
-	parts = append(parts, "{{.Message}}")
+
+	parts = append(parts, elems.Message)
 	return strings.Join(parts, " ")
 }
 
 // formatLevel aligns level to 5 chars
 func (l *Logger) formatLevel(lv string) string {
 
-	if lv == "" {
-		return ""
-	}
-
 	spaces := ""
 	if len(lv) == 4 {
 		spaces = " "
@@ -305,70 +313,3 @@ func getDump() []byte {
 	}
 	return stacktrace[:length]
 }
-
-// Option func type
-type Option func(l *Logger)
-
-// Out sets out writer, stdout by default
-func Out(w io.Writer) Option {
-	return func(l *Logger) {
-		l.stdout = w
-	}
-}
-
-// Err sets error writer, stderr by default
-func Err(w io.Writer) Option {
-	return func(l *Logger) {
-		l.stderr = w
-	}
-}
-
-// Debug turn on dbg mode
-func Debug(l *Logger) {
-	l.dbg = true
-}
-
-// Trace turn on trace + dbg mode
-func Trace(l *Logger) {
-	l.dbg = true
-	l.trace = true
-}
-
-// CallerDepth sets number of stack frame skipped for caller reporting, 0 by default
-func CallerDepth(n int) Option {
-	return func(l *Logger) {
-		l.callerDepth = n
-	}
-}
-
-// Format sets output layout, overwrites all options for individual parts, i.e. Caller*, Msec and LevelBraces
-func Format(f string) Option {
-	return func(l *Logger) {
-		l.format = f
-	}
-}
-
-// CallerFunc adds caller info with function name. Ignored if Format option used.
-func CallerFunc(l *Logger) {
-	l.callerFunc = true
-}
-
-// CallerPkg adds caller's package name. Ignored if Format option used.
-func CallerPkg(l *Logger) {
-	l.callerPkg = true
-}
-
-// LevelBraces surrounds level with [], i.e. [INFO]. Ignored if Format option used.
-func LevelBraces(l *Logger) {
-	l.levelBraces = true
-}
-
-// CallerFile adds caller info with file, and line number. Ignored if Format option used.
-func CallerFile(l *Logger) {
-	l.callerFile = true
-}
-
-// Msec adds .msec to timestamp. Ignored if Format option used.
-func Msec(l *Logger) {
-	l.msec = true
-}
diff --git a/backend/vendor/github.com/go-pkgz/lgr/options.go b/backend/vendor/github.com/go-pkgz/lgr/options.go
new file mode 100644
index 00000000..3d790395
--- /dev/null
+++ b/backend/vendor/github.com/go-pkgz/lgr/options.go
@@ -0,0 +1,70 @@
+package lgr
+
+import "io"
+
+// Option func type
+type Option func(l *Logger)
+
+// Out sets out writer, stdout by default
+func Out(w io.Writer) Option {
+	return func(l *Logger) {
+		l.stdout = w
+	}
+}
+
+// Err sets error writer, stderr by default
+func Err(w io.Writer) Option {
+	return func(l *Logger) {
+		l.stderr = w
+	}
+}
+
+// Debug turn on dbg mode
+func Debug(l *Logger) {
+	l.dbg = true
+}
+
+// Trace turn on trace + dbg mode
+func Trace(l *Logger) {
+	l.dbg = true
+	l.trace = true
+}
+
+// CallerDepth sets number of stack frame skipped for caller reporting, 0 by default
+func CallerDepth(n int) Option {
+	return func(l *Logger) {
+		l.callerDepth = n
+	}
+}
+
+// Format sets output layout, overwrites all options for individual parts, i.e. Caller*, Msec and LevelBraces
+func Format(f string) Option {
+	return func(l *Logger) {
+		l.format = f
+	}
+}
+
+// CallerFunc adds caller info with function name. Ignored if Format option used.
+func CallerFunc(l *Logger) {
+	l.callerFunc = true
+}
+
+// CallerPkg adds caller's package name. Ignored if Format option used.
+func CallerPkg(l *Logger) {
+	l.callerPkg = true
+}
+
+// LevelBraces surrounds level with [], i.e. [INFO]. Ignored if Format option used.
+func LevelBraces(l *Logger) {
+	l.levelBraces = true
+}
+
+// CallerFile adds caller info with file, and line number. Ignored if Format option used.
+func CallerFile(l *Logger) {
+	l.callerFile = true
+}
+
+// Msec adds .msec to timestamp. Ignored if Format option used.
+func Msec(l *Logger) {
+	l.msec = true
+}
diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt
index bdc6b4eb..55764647 100644
--- a/backend/vendor/modules.txt
+++ b/backend/vendor/modules.txt
@@ -23,7 +23,7 @@ github.com/globalsign/mgo/bson
 github.com/globalsign/mgo/internal/sasl
 github.com/globalsign/mgo/internal/scram
 github.com/globalsign/mgo/internal/json
-# github.com/go-chi/chi v3.3.2+incompatible
+# github.com/go-chi/chi v4.0.2+incompatible
 github.com/go-chi/chi
 github.com/go-chi/chi/middleware
 # github.com/go-chi/cors v1.0.0
@@ -39,7 +39,7 @@ github.com/go-pkgz/auth/logger
 github.com/go-pkgz/auth/middleware
 # github.com/go-pkgz/lcw v0.2.0
 github.com/go-pkgz/lcw
-# github.com/go-pkgz/lgr v0.6.1
+# github.com/go-pkgz/lgr v0.6.2
 github.com/go-pkgz/lgr
 # github.com/go-pkgz/mongo v1.1.2
 github.com/go-pkgz/mongo

From 0de9a7b0d652137442b82c03cea8ce0e3670776b Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Thu, 4 Apr 2019 09:37:28 -0500
Subject: [PATCH 37/45] set longer time to start integration server test

---
 backend/app/main_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/app/main_test.go b/backend/app/main_test.go
index d9385171..0409ed95 100644
--- a/backend/app/main_test.go
+++ b/backend/app/main_test.go
@@ -16,13 +16,13 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestMain(t *testing.T) {
+func Test_Main(t *testing.T) {
 
 	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
 		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(500 * time.Millisecond)
+		time.Sleep(1000 * time.Millisecond)
 		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
 		require.Nil(t, err)
 	}()
@@ -32,11 +32,11 @@ func TestMain(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 1, "should take about 500msec")
+		assert.True(t, time.Since(st).Seconds() < 2, "should take under 1s")
 		wg.Done()
 	}()
 
-	time.Sleep(200 * time.Millisecond) // let server start
+	time.Sleep(500 * time.Millisecond) // let server start
 
 	// send ping
 	resp, err := http.Get("http://localhost:18202/api/v1/ping")

From ad3430883f99931ecabddf02dfd189ce15a04c8f Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Thu, 4 Apr 2019 13:54:23 -0500
Subject: [PATCH 38/45] make temp for integration tests

---
 backend/app/main_test.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/backend/app/main_test.go b/backend/app/main_test.go
index 0409ed95..edc2e7e5 100644
--- a/backend/app/main_test.go
+++ b/backend/app/main_test.go
@@ -18,8 +18,12 @@ import (
 
 func Test_Main(t *testing.T) {
 
-	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=/tmp/xyz", "--backup=/tmp",
-		"--avatar.fs.path=/tmp", "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+	dir, err := ioutil.TempDir(os.TempDir(), "remark42")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
+		"--avatar.fs.path=" + dir, "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
 		time.Sleep(1000 * time.Millisecond)

From 50e437c22c1c822e19c1674e509afd73fc8fa7e2 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Thu, 4 Apr 2019 22:51:41 -0500
Subject: [PATCH 39/45] switch test port

---
 backend/app/main_test.go | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/backend/app/main_test.go b/backend/app/main_test.go
index edc2e7e5..392985a1 100644
--- a/backend/app/main_test.go
+++ b/backend/app/main_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/repeater"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -23,12 +25,12 @@ func Test_Main(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	os.Args = []string{"test", "server", "--secret=123456", "--store.bolt.path=" + dir, "--backup=/tmp",
-		"--avatar.fs.path=" + dir, "--port=18202", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
+		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
 		time.Sleep(1000 * time.Millisecond)
-		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
-		require.Nil(t, err)
+		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		require.Nil(t, e)
 	}()
 
 	wg := sync.WaitGroup{}
@@ -40,16 +42,25 @@ func Test_Main(t *testing.T) {
 		wg.Done()
 	}()
 
-	time.Sleep(500 * time.Millisecond) // let server start
+	var passed bool
+	err = repeater.NewDefault(10, time.Millisecond*100).Do(context.Background(), func() error {
+		resp, e := http.Get("http://localhost:18222/api/v1/ping")
+		if e != nil {
+			t.Logf("%+v", e)
+			return e
+		}
+		require.Nil(t, e)
+		defer resp.Body.Close()
+		assert.Equal(t, 200, resp.StatusCode)
+		body, e := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, e)
+		assert.Equal(t, "pong", string(body))
+		passed = true
+		return nil
+	})
 
-	// send ping
-	resp, err := http.Get("http://localhost:18202/api/v1/ping")
-	require.Nil(t, err)
-	defer resp.Body.Close()
-	assert.Equal(t, 200, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	assert.Nil(t, err)
-	assert.Equal(t, "pong", string(body))
+	assert.NoError(t, err)
+	assert.Equal(t, true, passed)
 
 	wg.Wait()
 }

From 5d052f8eff9f7d97a2646b95866f48220ab3fe88 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 5 Apr 2019 16:24:01 -0500
Subject: [PATCH 40/45] remove pkg info for prod logging, add braces

---
 backend/app/main.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/app/main.go b/backend/app/main.go
index 4e41a3d6..c49963ab 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -62,10 +62,10 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec)
+		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
 		return
 	}
-	log.Setup(log.Msec, log.CallerPkg)
+	log.Setup(log.Msec, log.LevelBraces)
 }
 
 // getDump reads runtime stack and returns as a string

From bf17b31eaa00970d383c04e4b1a65e9fef12628b Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 5 Apr 2019 16:24:58 -0500
Subject: [PATCH 41/45] add caller func to debug logging

---
 backend/app/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/main.go b/backend/app/main.go
index c49963ab..5e00c160 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -62,7 +62,7 @@ func main() {
 
 func setupLog(dbg bool) {
 	if dbg {
-		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
+		log.Setup(log.Debug, log.CallerFile, log.CallerFunc, log.Msec, log.LevelBraces)
 		return
 	}
 	log.Setup(log.Msec, log.LevelBraces)

From e1d4c9ef2344bd9782db35f307af5d87b006af55 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 5 Apr 2019 16:42:36 -0500
Subject: [PATCH 42/45] add progress debug for bolt creation

---
 backend/app/store/engine/bolt_accessor.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 22a62893..664a0e40 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -65,6 +65,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
 					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
 				}
+				log.Printf("[DEBUG] created bucket %s", string(bktName))
 			}
 			return nil
 		})
@@ -74,6 +75,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		result.dbs[site.SiteID] = db
+		log.Printf("[DEBUG] bolt store created for %+v", site.SiteID)
 	}
 	return &result, nil
 }

From 47414a252f508c8c108695d69719e9945af386d8 Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 5 Apr 2019 16:47:44 -0500
Subject: [PATCH 43/45] increase main test time to address crazy CI (travis)
 slowness in making buckets

---
 backend/app/main_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/app/main_test.go b/backend/app/main_test.go
index 392985a1..219f9b73 100644
--- a/backend/app/main_test.go
+++ b/backend/app/main_test.go
@@ -28,7 +28,7 @@ func Test_Main(t *testing.T) {
 		"--avatar.fs.path=" + dir, "--port=18222", "--url=https://demo.remark42.com", "--dbg", "--notify.type=none"}
 
 	go func() {
-		time.Sleep(1000 * time.Millisecond)
+		time.Sleep(2000 * time.Millisecond)
 		e := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
 		require.Nil(t, e)
 	}()
@@ -38,12 +38,12 @@ func Test_Main(t *testing.T) {
 	go func() {
 		st := time.Now()
 		main()
-		assert.True(t, time.Since(st).Seconds() < 2, "should take under 1s")
+		assert.True(t, time.Since(st).Seconds() > 2, "should take 2s")
 		wg.Done()
 	}()
 
 	var passed bool
-	err = repeater.NewDefault(10, time.Millisecond*100).Do(context.Background(), func() error {
+	err = repeater.NewDefault(10, time.Millisecond*200).Do(context.Background(), func() error {
 		resp, e := http.Get("http://localhost:18222/api/v1/ping")
 		if e != nil {
 			t.Logf("%+v", e)

From 3662cd0ab5c4286284b265fb132bba6137c0bb6f Mon Sep 17 00:00:00 2001
From: Umputun <umputun@gmail.com>
Date: Fri, 5 Apr 2019 16:49:42 -0500
Subject: [PATCH 44/45] lint: convert string to string

---
 backend/app/store/engine/bolt_accessor.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 664a0e40..2348958e 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -65,7 +65,6 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 				if _, e := tx.CreateBucketIfNotExists([]byte(bktName)); e != nil {
 					return errors.Wrapf(e, "failed to create top level bucket %s", bktName)
 				}
-				log.Printf("[DEBUG] created bucket %s", string(bktName))
 			}
 			return nil
 		})
@@ -75,7 +74,7 @@ func NewBoltDB(options bolt.Options, sites ...BoltSite) (*BoltDB, error) {
 		}
 
 		result.dbs[site.SiteID] = db
-		log.Printf("[DEBUG] bolt store created for %+v", site.SiteID)
+		log.Printf("[DEBUG] bolt store created for %s", site.SiteID)
 	}
 	return &result, nil
 }

From db2c1c0c35124a140053cc12f5d18ed9674b9502 Mon Sep 17 00:00:00 2001
From: ns-cweber <cweber@narrativescience.com>
Date: Sun, 7 Apr 2019 10:19:35 -0500
Subject: [PATCH 45/45] Fix formatting in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b449f9bc..4055096b 100644
--- a/README.md
+++ b/README.md
@@ -103,7 +103,7 @@ _this is the recommended way to run remark42_
 | cache.max.items         | CACHE_MAX_ITEMS         | `1000`                   | max number of cached items, `0` - unlimited      |
 | cache.max.value         | CACHE_MAX_VALUE         | `65536`                  | max size of cached value, `0` - unlimited        |
 | cache.max.size          | CACHE_MAX_SIZE          | `50000000`               | max size of all cached values, `0` - unlimited   |
-| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, 'bolt`, or `mongo` |
+| avatar.type             | AVATAR_TYPE             | `fs`                     | type of avatar storage, `fs`, `bolt`, or `mongo` |
 | avatar.fs.path          | AVATAR_FS_PATH          | `./var/avatars`          | avatars location for `fs` store                  |
 | avatar.bolt.file        | AVATAR_BOLT_FILE        | `./var/avatars.db`       | file name for  `bolt` store                      |
 | avatar.rsz-lmt          | AVATAR_RSZ_LMT          | `0` (disabled)           | max image size for resizing avatars on save      |