Define remark host in runtime (#325)

* convert closest-polyfill to ts

* make remark respect host property in client setup

Dockerfile

@@ -25,12 +25,14 @@ ADD backend /build/backend
 ADD .git /build/.git
 WORKDIR /build/backend
 
+ENV GOFLAGS="-mod=vendor"
+
 # run tests
 RUN \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -mod=vendor -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
@@ -55,7 +57,7 @@ RUN \
     if [ -z "$DRONE" ] ; then echo "runs outside of drone" && version="local"; \
     else version=${DRONE_TAG}${DRONE_BRANCH}${DRONE_PULL_REQUEST}-${DRONE_COMMIT:0:7}-$(date +%Y%m%d-%H:%M:%S); fi && \
     echo "version=$version" && \
-    go build -mod=vendor -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
+    go build -o remark42 -ldflags "-X main.revision=${version} -s -w" ./app
 
 
 FROM node:10.11-alpine as build-frontend-deps

README.md

@@ -308,18 +308,28 @@ Add this snippet to the bottom of web page:
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['embed'] // optional param; which components to load. default to ["embed"]
+                          // to load all components define components as ['embed', 'last-comments', 'counter']
+                          // available component are:
+                          //     - 'embed': basic comments widget
+                          //     - 'last-comments': last comments widget, see `Last Comments` section below
+                          //     - 'counter': counter widget, see `Counter` section below
     url: 'PAGE_URL', // optional param; if it isn't defined window.location.href will be used
     max_shown_comments: 10, // optional param; if it isn't defined default value (15) will be used
     theme: 'dark', // optional param; if it isn't defined default value ('light') will be used
     page_title: 'Moving to Remark42' // optional param; if it isn't defined `document.title` will be used
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/embed.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -348,19 +358,24 @@ window.REMARK42.changeTheme('light');
 
 It's a widget which renders list of last comments from your site.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `last-comments` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['last-comments']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/last-comments.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 
@@ -376,19 +391,24 @@ And then add this node in the place where you want to see last comments widget:
 
 It's a widget which renders a number of comments for the specified page.
 
-Add this snippet to the bottom of web page:
+Add this snippet to the bottom of web page, or adjust already present `remark_config` to have `counter` in `components` list:
 
 ```html
 <script>
   var remark_config = {
+    host: "REMARK_URL", // hostname of remark server, same as REMARK_URL in backend config, e.g. "https://demo.remark42.com"
     site_id: 'YOUR_SITE_ID',
+    components: ['counter']
   };
 
-  (function() {
-    var d = document, s = d.createElement('script');
-    s.src = '/web/counter.js'; // prepends this address with domain where remark42 is placed
-    (d.head || d.body).appendChild(s);
-  })();
+  (function(c) {
+    for(var i = 0; i < c.length; i++){
+      var d = document, s = d.createElement('script');
+      s.src = remark_config.host + '/web/' +c[i] +'.js';
+      s.defer = true;
+      (d.head || d.body).appendChild(s);
+    }
+  })(remark_config.components || ['embed']);
 </script>
 ```
 

backend/app/cmd/cleanup.go

@@ -131,7 +131,7 @@ func (cc *CleanupCommand) postsInRange(fromS, toS string) ([]store.PostInfo, err
 // get all posts via GET /list?site=siteID&limit=50&skip=10
 func (cc *CleanupCommand) listPosts() ([]store.PostInfo, error) {
 	listURL := fmt.Sprintf("%s/api/v1/list?site=%s&limit=10000", cc.RemarkURL, cc.Site)
-	r, err := http.Get(listURL) //nolint
+	r, err := http.Get(listURL) // nolint
 	if err != nil {
 		return nil, errors.Wrapf(err, "get request failed for list of posts, site %s", cc.Site)
 	}
@@ -158,7 +158,7 @@ func (cc *CleanupCommand) listComments(postURL string) ([]store.Comment, error)
 
 	// handle 429 error from limiter
 	for {
-		r, err = http.Get(commentsURL) //nolint
+		r, err = http.Get(commentsURL) // nolint
 		if err != nil {
 			return nil, errors.Wrapf(err, "get request failed for comments, %s", postURL)
 		}

backend/app/cmd/server.go

@@ -630,6 +630,7 @@ func (s *ServerCommand) makeAuthenticator(ds *service.DataStore, avas avatar.Sto
 			}
 			return !claims.User.BoolAttr("blocked")
 		}),
+		JWTQuery:          "jwt", // change default from "token" as it used for deleteme
 		AvatarStore:       avas,
 		AvatarResizeLimit: s.Avatar.RszLmt,
 		AvatarRoutePath:   "/api/v1/avatar",

backend/app/cmd/server_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
@@ -12,20 +13,21 @@ import (
 	"testing"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/globalsign/mgo"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/mongo"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestServerApp(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 1500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+		o.Port = port
 		return o
 	})
 
@@ -33,7 +35,7 @@ func TestServerApp(t *testing.T) {
 	time.Sleep(100 * time.Millisecond) // let server start
 
 	// send ping
-	resp, err := http.Get("http://localhost:18080/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -43,7 +45,7 @@ func TestServerApp(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 5 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
@@ -59,8 +61,9 @@ func TestServerApp(t *testing.T) {
 }
 
 func TestServerApp_DevMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.AdminPasswd = "password"
 		o.Auth.Dev = true
 		return o
@@ -72,7 +75,7 @@ func TestServerApp_DevMode(t *testing.T) {
 	assert.Equal(t, 4+1, len(app.restSrv.Authenticator.Providers()), "extra auth provider")
 	assert.Equal(t, "dev", app.restSrv.Authenticator.Providers()[4].Name(), "dev auth provider")
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -84,8 +87,9 @@ func TestServerApp_DevMode(t *testing.T) {
 }
 
 func TestServerApp_AnonMode(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18085
+		o.Port = port
 		o.Auth.Anonymous = true
 		return o
 	})
@@ -97,7 +101,7 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "anonymous", app.restSrv.Authenticator.Providers()[4].Name(), "anon auth provider")
 
 	// send ping
-	resp, err := http.Get("http://localhost:18085/api/v1/ping")
+	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/ping", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
@@ -106,19 +110,19 @@ func TestServerApp_AnonMode(t *testing.T) {
 	assert.Equal(t, "pong", string(body))
 
 	// try to login with good name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=blah123&aud=remark42")
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=blah123&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// try to login with bad name
-	resp, err = http.Get("http://localhost:18085/auth/anonymous/login?user=**blah123&aud=remark42")
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=**blah123&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 403, resp.StatusCode)
 
 	// try to login with short name
-	resp, err = http.Get(`http://localhost:18085/auth/anonymous/login?user=bl%20%20&aud=remark42`)
+	resp, err = http.Get(fmt.Sprintf("http://localhost:%d/auth/anonymous/login?user=bl%20%20&aud=remark42", port))
 	require.Nil(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, 403, resp.StatusCode)
@@ -284,7 +288,7 @@ func TestServerApp_Failed(t *testing.T) {
 
 func TestServerApp_Shutdown(t *testing.T) {
 	app, ctx := prepServerApp(t, 500*time.Millisecond, func(o ServerCommand) ServerCommand {
-		o.Port = 18090
+		o.Port = rand.Intn(40000) + 10000
 		return o
 	})
 	st := time.Now()
@@ -360,8 +364,9 @@ func Test_ACMEEmail(t *testing.T) {
 }
 
 func TestServerAuthHooks(t *testing.T) {
+	port := rand.Intn(40000) + 10000
 	app, ctx := prepServerApp(t, 5*time.Second, func(o ServerCommand) ServerCommand {
-		o.Port = 18080
+		o.Port = port
 		return o
 	})
 
@@ -390,7 +395,7 @@ func TestServerAuthHooks(t *testing.T) {
 
 	// add comment
 	client := http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123", "locator":{"url": "https://radio-t.com/p/2018/12/29/podcast-630/", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
 	require.Nil(t, err)
@@ -400,7 +405,8 @@ func TestServerAuthHooks(t *testing.T) {
 	assert.Equal(t, http.StatusCreated, resp.StatusCode, "non-blocked user able to post")
 
 	// block user dev as admin
-	req, e := http.NewRequest(http.MethodPut, "http://localhost:18080/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", nil)
+	req, e := http.NewRequest(http.MethodPut,
+		fmt.Sprintf("http://localhost:%d/api/v1/admin/user/dev?site=remark&block=1&ttl=10d", port), nil)
 	assert.Nil(t, e)
 	req.SetBasicAuth("admin", "password")
 	resp, e = client.Do(req)
@@ -414,7 +420,7 @@ func TestServerAuthHooks(t *testing.T) {
 	time.Sleep(2 * time.Second) // make sure token expired and refresh happened
 
 	// try add a comment with blocked user
-	req, err = http.NewRequest("POST", "http://localhost:18080/api/v1/comment",
+	req, err = http.NewRequest("POST", fmt.Sprintf("http://localhost:%d/api/v1/comment", port),
 		strings.NewReader(`{"text": "test 123 blah", "locator":{"url": "https://radio-t.com/blah1", "site": "remark"}}`))
 	req.Header.Set("X-JWT", tk)
 	require.Nil(t, err)
@@ -459,5 +465,6 @@ func prepServerApp(t *testing.T, duration time.Duration, fn func(o ServerCommand
 		log.Print("[TEST] terminate app")
 		cancel()
 	}()
+	rand.Seed(time.Now().UnixNano())
 	return app, ctx
 }

backend/app/main.go

@@ -8,7 +8,7 @@ import (
 	"syscall"
 
 	log "github.com/go-pkgz/lgr"
-	flags "github.com/jessevdk/go-flags"
+	"github.com/jessevdk/go-flags"
 
 	"github.com/umputun/remark/backend/app/cmd"
 )

backend/app/migrator/wordpress_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/umputun/remark/backend/app/store"

backend/app/notify/telegram.go

@@ -113,10 +113,10 @@ func (t *Telegram) Send(ctx context.Context, req request) error {
 	}
 
 	r, err := http.NewRequest("POST", u, bytes.NewReader(b))
-	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 	if err != nil {
 		return errors.Wrap(err, "failed to make telegram request")
 	}
+	r.Header.Set("Content-Type", "application/json; charset=utf-8")
 
 	r = r.WithContext(ctx)
 	resp, err := client.Do(r)

backend/app/rest/api/admin.go

@@ -27,25 +27,6 @@ type admin struct {
 	migrator      *Migrator
 }
 
-func (a *admin) routes(middlewares ...func(http.Handler) http.Handler) chi.Router {
-	router := chi.NewRouter()
-	router.Use(middlewares...)
-	router.Delete("/comment/{id}", a.deleteCommentCtrl)
-	router.Put("/user/{userid}", a.setBlockCtrl)
-	router.Delete("/user/{userid}", a.deleteUserCtrl)
-	router.Get("/user/{userid}", a.getUserInfoCtrl)
-	router.Get("/deleteme", a.deleteMeRequestCtrl)
-	router.Put("/verify/{userid}", a.setVerifyCtrl)
-	router.Put("/pin/{id}", a.setPinCtrl)
-	router.Get("/blocked", a.blockedUsersCtrl)
-	router.Put("/readonly", a.setReadOnlyCtrl)
-	router.Put("/title/{id}", a.setTitleCtrl)
-
-	a.migrator.withRoutes(router) // set migrator routes, i.e. /export and /import
-
-	return router
-}
-
 // DELETE /comment/{id}?site=siteID&url=post-url - removes comment
 func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 

backend/app/rest/api/admin_test.go

@@ -58,12 +58,11 @@ func TestAdmin_Delete(t *testing.T) {
 		{URL: "https://radio-t.com/blah2", Count: 0}}), j)
 
 	// delete a comment
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete,
 		fmt.Sprintf("%s/api/v1/admin/comment/%s?site=radio-t&url=https://radio-t.com/blah", ts.URL, id1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	requireAdminOnly(t, req)
+	resp, err = sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
@@ -134,13 +133,12 @@ func TestAdmin_Title(t *testing.T) {
 	id1 := addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/title/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	body, code := get(t, fmt.Sprintf("%s/api/v1/id/%s?site=radio-t&url=%s/post1", ts.URL, id1, tss.URL))
@@ -170,38 +168,37 @@ func TestAdmin_DeleteUser(t *testing.T) {
 	_, err = srv.DataService.Create(c3)
 	assert.NoError(t, err)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t", ts.URL, "id2"), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
 	assert.Nil(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	// all 3 comments here, but for id2 they deleted
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah&sort=+time")
 	assert.Equal(t, 200, code)
-	commentsWithInfo := commentsWithInfo{}
-	err = json.Unmarshal([]byte(res), &commentsWithInfo)
+	cmntWithInfo := commentsWithInfo{}
+	err = json.Unmarshal([]byte(res), &cmntWithInfo)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(commentsWithInfo.Comments), "should have 3 comment")
+	assert.Equal(t, 3, len(cmntWithInfo.Comments), "should have 3 comment")
 
 	// id1 comment untouched
-	assert.Equal(t, id1, commentsWithInfo.Comments[0].ID)
-	assert.Equal(t, "o test test #1", commentsWithInfo.Comments[0].Orig)
-	assert.False(t, commentsWithInfo.Comments[0].Deleted)
-	t.Logf("%+v", commentsWithInfo.Comments[0].User)
+	assert.Equal(t, id1, cmntWithInfo.Comments[0].ID)
+	assert.Equal(t, "o test test #1", cmntWithInfo.Comments[0].Orig)
+	assert.False(t, cmntWithInfo.Comments[0].Deleted)
+	t.Logf("%+v", cmntWithInfo.Comments[0].User)
 
 	// id2 comments fully deleted
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[1].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[1].Deleted)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[1].Orig)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
+	assert.True(t, cmntWithInfo.Comments[1].Deleted)
 
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Text)
-	assert.Equal(t, "", commentsWithInfo.Comments[2].Orig)
-	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, commentsWithInfo.Comments[1].User)
-	assert.True(t, commentsWithInfo.Comments[2].Deleted)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Text)
+	assert.Equal(t, "", cmntWithInfo.Comments[2].Orig)
+	assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, cmntWithInfo.Comments[1].User)
+	assert.True(t, cmntWithInfo.Comments[2].Deleted)
 }
 
 func TestAdmin_Pin(t *testing.T) {
@@ -221,6 +218,7 @@ func TestAdmin_Pin(t *testing.T) {
 		req, err := http.NewRequest(http.MethodPut,
 			fmt.Sprintf("%s/api/v1/admin/pin/%s?site=radio-t&url=https://radio-t.com/blah&pin=%d", ts.URL, id1, val), nil)
 		assert.Nil(t, err)
+		requireAdminOnly(t, req)
 		req.SetBasicAuth("admin", "password")
 		resp, err := client.Do(req)
 		assert.Nil(t, err)
@@ -262,15 +260,14 @@ func TestAdmin_Block(t *testing.T) {
 	assert.Nil(t, err)
 
 	block := func(val int, ttl string) (code int, body []byte) {
-		client := http.Client{}
 		url := fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", val)
 		if ttl != "" {
 			url = url + "&ttl=" + ttl
 		}
 		req, e := http.NewRequest(http.MethodPut, url, nil)
 		assert.Nil(t, e)
-		req.SetBasicAuth("admin", "password")
-		resp, e := client.Do(req)
+		requireAdminOnly(t, req)
+		resp, e := sendReq(t, req, adminUmputunToken)
 		require.Nil(t, e)
 		body, e = ioutil.ReadAll(resp.Body)
 		assert.Nil(t, e)
@@ -332,29 +329,26 @@ func TestAdmin_BlockedList(t *testing.T) {
 	ts, _, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
-
 	// block user1
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d", ts.URL, "user1", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	res, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	// block user2
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/user/%s?site=radio-t&block=%d&ttl=50ms", ts.URL, "user2", 1), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, res.StatusCode)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	res, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users := []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
@@ -366,10 +360,9 @@ func TestAdmin_BlockedList(t *testing.T) {
 	time.Sleep(50 * time.Millisecond)
 
 	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/blocked?site=radio-t", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	res, err = client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	res, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 	users = []store.BlockedUser{}
 	err = json.NewDecoder(res.Body).Decode(&users)
@@ -396,15 +389,15 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	resp, err := sendReq(t, req, "") // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
@@ -416,20 +409,18 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err := json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusForbidden, resp.StatusCode)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.Nil(t, err)
 	assert.False(t, info.ReadOnly)
@@ -440,10 +431,9 @@ func TestAdmin_ReadOnly(t *testing.T) {
 	b, err = json.Marshal(c)
 	assert.Nil(t, err, "can't marshal comment %+v", c)
 	req, err = http.NewRequest("POST", ts.URL+"/api/v1/comment", bytes.NewBuffer(b))
-	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
-	assert.Nil(t, err)
+	require.NoError(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, http.StatusCreated, resp.StatusCode)
 }
 
@@ -451,15 +441,13 @@ func TestAdmin_ReadOnlyNoComments(t *testing.T) {
 	ts, srv, teardown := startupT(t)
 	defer teardown()
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	_, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
 	assert.NotNil(t, err)
@@ -488,30 +476,27 @@ func TestAdmin_ReadOnlyWithAge(t *testing.T) {
 	assert.Nil(t, err)
 	assert.True(t, info.ReadOnly, "ro by age")
 
-	client := http.Client{}
-
 	// set post to read-only
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 	// reset post's read-only
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/readonly?site=radio-t&url=https://radio-t.com/blah&ro=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err = client.Do(req)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 	assert.Equal(t, 403, resp.StatusCode)
-	require.Nil(t, err)
 	info, err = srv.DataService.Info(store.Locator{SiteID: "radio-t", URL: "https://radio-t.com/blah"}, 0)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, info.ReadOnly)
 
 }
@@ -532,13 +517,13 @@ func TestAdmin_Verify(t *testing.T) {
 	verified := srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
-	client := http.Client{}
 	req, err := http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=1", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.True(t, verified)
 
@@ -554,9 +539,9 @@ func TestAdmin_Verify(t *testing.T) {
 	req, err = http.NewRequest(http.MethodPut,
 		fmt.Sprintf("%s/api/v1/admin/verify/user1?site=radio-t&verified=0", ts.URL), nil)
 	assert.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	_, err = client.Do(req)
-	require.Nil(t, err)
+	resp, err = sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
 	verified = srv.DataService.IsVerified("radio-t", "user1")
 	assert.False(t, verified)
 
@@ -601,12 +586,11 @@ func TestAdmin_ExportFile(t *testing.T) {
 	addComment(t, c1, ts)
 	addComment(t, c2, ts)
 
-	client := &http.Client{Timeout: 5 * time.Second}
 	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t&mode=file", nil)
-	require.Nil(t, err)
-	req.SetBasicAuth("admin", "password")
-	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
+	requireAdminOnly(t, req)
+	resp, err := sendReq(t, req, adminUmputunToken)
+	require.NoError(t, err)
 
 	assert.Equal(t, 200, resp.StatusCode)
 	assert.Equal(t, "application/gzip", resp.Header.Get("Content-Type"))
@@ -665,9 +649,10 @@ func TestAdmin_DeleteMeRequest(t *testing.T) {
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/admin/deleteme?token=%s", ts.URL, tkn), nil)
 	assert.Nil(t, err)
+
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 
 	_, err = srv.DataService.User("radio-t", "user1", 0, 0)

backend/app/rest/api/migrator.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -40,15 +39,6 @@ type KeyStore interface {
 	Key() (key string, err error)
 }
 
-func (m *Migrator) withRoutes(router chi.Router) chi.Router {
-	router.Get("/export", m.exportCtrl)
-
-	router.Post("/import", m.importCtrl)
-	router.Post("/import/form", m.importFormCtrl)
-	router.Get("/import/wait", m.importWaitCtrl)
-	return router
-}
-
 // POST /import?secret=key&site=site-id&provider=disqus|remark|wordpress
 // imports comments from post body.
 func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {

backend/app/rest/api/migrator_test.go

@@ -6,38 +6,26 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
-	"github.com/go-chi/chi"
-	"github.com/go-pkgz/auth"
-	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/rest/cache"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/umputun/remark/backend/app/migrator"
-	adminstore "github.com/umputun/remark/backend/app/store/admin"
-	"github.com/umputun/remark/backend/app/store/engine"
-	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestMigrator_Import(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -52,7 +40,7 @@ func TestMigrator_Import(t *testing.T) {
 }
 
 func TestMigrator_ImportForm(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
@@ -68,7 +56,7 @@ func TestMigrator_ImportForm(t *testing.T) {
 	require.NoError(t, bodyWriter.Close())
 
 	authts := strings.Replace(ts.URL, "http://", "http://admin:password@", 1)
-	resp, err := http.Post(authts+"/import/form?site=radio-t&provider=native", contentType, bodyBuf)
+	resp, err := http.Post(authts+"/api/v1/admin/import/form?site=radio-t&provider=native", contentType, bodyBuf)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
@@ -80,13 +68,13 @@ func TestMigrator_ImportForm(t *testing.T) {
 }
 
 func TestMigrator_ImportFromWP(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(strings.Replace(xmlTestWP, "'", "`", -1))
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=wordpress", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=wordpress", r)
 	assert.Nil(t, err)
 	req.Header.Add("Content-Type", "application/xml; charset=utf-8")
 	req.SetBasicAuth("admin", "password")
@@ -102,14 +90,14 @@ func TestMigrator_ImportFromWP(t *testing.T) {
 }
 
 func TestMigrator_ImportRejected(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
 	{"id":"83fd97fd-ff64-48d1-9fb7-ca7769c77037","pid":"p1","text":"<p>test test #2</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah2"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.861387771-05:00"}`)
 
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native&secret=XYZ", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native&secret=XYZ", r)
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
 	assert.Nil(t, err)
@@ -117,7 +105,7 @@ func TestMigrator_ImportRejected(t *testing.T) {
 }
 
 func TestMigrator_ImportDouble(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
@@ -127,7 +115,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err := client.Do(req)
@@ -135,7 +123,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 1 * time.Second}
-	req, err = http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err = http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	assert.Nil(t, err)
 	resp, err = client.Do(req)
@@ -145,7 +133,7 @@ func TestMigrator_ImportDouble(t *testing.T) {
 }
 
 func TestMigrator_ImportWaitExpired(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	tmpl := `{"id":"%d","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}`
@@ -155,7 +143,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	}
 	r := strings.NewReader(`{"version":1}` + strings.Join(recs, "\n")) // reader with 10k records
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	req.SetBasicAuth("admin", "password")
 	require.Nil(t, err)
 	resp, err := client.Do(req)
@@ -163,7 +151,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 	assert.Equal(t, http.StatusAccepted, resp.StatusCode)
 
 	client = &http.Client{Timeout: 10 * time.Second}
-	req, err = http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t&timeout=100ms", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t&timeout=100ms", nil)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err = client.Do(req)
@@ -174,7 +162,7 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
 }
 
 func TestMigrator_Export(t *testing.T) {
-	ts, teardown := prepImportSrv(t)
+	ts, _, teardown := startupT(t)
 	defer teardown()
 
 	r := strings.NewReader(`{"version":1} {"id":"2aa0478c-df1b-46b1-b561-03d507cf482c","pid":"","text":"<p>test test #1</p>","user":{"name":"developer one","id":"dev","picture":"/api/v1/avatar/remark.image","profile":"https://remark42.com","admin":true,"ip":"ae12fe3b5f129b5cc4cdd2b136b7b7947c4d2741"},"locator":{"site":"radio-t","url":"https://radio-t.com/blah1"},"score":0,"votes":{},"time":"2018-04-30T01:37:00.849053725-05:00"}
@@ -182,7 +170,7 @@ func TestMigrator_Export(t *testing.T) {
 
 	// import comments first
 	client := &http.Client{Timeout: 1 * time.Second}
-	req, err := http.NewRequest("POST", ts.URL+"/import?site=radio-t&provider=native", r)
+	req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=radio-t&provider=native", r)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err := client.Do(req)
@@ -191,7 +179,7 @@ func TestMigrator_Export(t *testing.T) {
 	waitForImportCompletion(t, ts)
 
 	// check file mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=file&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=file&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -208,7 +196,7 @@ func TestMigrator_Export(t *testing.T) {
 	t.Logf("%s", string(ungzBody))
 
 	// check stream mode
-	req, err = http.NewRequest("GET", ts.URL+"/export?mode=stream&site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?mode=stream&site=radio-t", nil)
 	require.Nil(t, err)
 	req.SetBasicAuth("admin", "password")
 	resp, err = client.Do(req)
@@ -222,7 +210,7 @@ func TestMigrator_Export(t *testing.T) {
 	assert.Equal(t, 2, strings.Count(string(body), "\"text\""))
 	t.Logf("%s", string(body))
 
-	req, err = http.NewRequest("GET", ts.URL+"/export?site=radio-t", nil)
+	req, err = http.NewRequest("GET", ts.URL+"/api/v1/admin/export?site=radio-t", nil)
 	require.Nil(t, err)
 	resp, err = client.Do(req)
 	require.Nil(t, err)
@@ -231,7 +219,7 @@ func TestMigrator_Export(t *testing.T) {
 
 func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	req, err := http.NewRequest("GET", ts.URL+"/import/wait?site=radio-t", nil)
+	req, err := http.NewRequest("GET", ts.URL+"/api/v1/admin/import/wait?site=radio-t", nil)
 	req.SetBasicAuth("admin", "password")
 	assert.NoError(t, err)
 	resp, err := client.Do(req)
@@ -243,38 +231,6 @@ func waitForImportCompletion(t *testing.T, ts *httptest.Server) {
 	assert.Equal(t, "{\"site_id\":\"radio-t\",\"status\":\"completed\"}\n", string(b))
 }
 
-func prepImportSrv(t *testing.T) (ts *httptest.Server, teardown func()) {
-	testDb := fmt.Sprintf("/tmp/test-remark-import-%d.db", rand.Int31())
-	b, err := engine.NewBoltDB(bolt.Options{}, engine.BoltSite{FileName: testDb, SiteID: "radio-t"})
-	require.Nil(t, err)
-	adminStore := adminstore.NewStaticStore("123456", []string{"a1", "a2"}, "admin@remark-42.com")
-	dataStore := &service.DataStore{Interface: b, AdminStore: adminStore}
-	svc := &Migrator{
-		DisqusImporter:    &migrator.Disqus{DataStore: dataStore},
-		WordPressImporter: &migrator.WordPress{DataStore: dataStore},
-		NativeImporter:    &migrator.Native{DataStore: dataStore},
-		NativeExporter:    &migrator.Native{DataStore: dataStore},
-		Cache:             &cache.Nop{},
-		KeyStore:          adminStore,
-	}
-
-	a := auth.NewService(auth.Opts{
-		AdminPasswd:  "password",
-		SecretReader: token.SecretFunc(func() (string, error) { return "123456", nil }),
-		Issuer:       "test",
-	})
-
-	am := a.Middleware()
-	routes := svc.withRoutes(chi.NewRouter().With(am.Auth).With(am.AdminOnly))
-	ts = httptest.NewServer(routes)
-	teardown = func() {
-		ts.Close()
-		require.NoError(t, b.Close())
-		_ = os.Remove(testDb)
-	}
-	return ts, teardown
-}
-
 var xmlTestWP = `
 <?xml version="1.0" encoding="UTF-8" ?>
 <rss version="2.0"

backend/app/rest/api/rest.go

@@ -5,9 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path"
 	"strings"
 	"sync"
 	"time"
@@ -17,7 +15,6 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/cors"
-	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
@@ -189,12 +186,12 @@ func (s *Rest) routes() chi.Router {
 	router.Use(corsMiddleware.Handler)
 
 	ipFn := func(ip string) string { return store.HashValue(ip, s.SharedSecret)[:12] } // logger uses it for anonymization
+	logInfoWithBody := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]")).Handler
 
 	authHandler, avatarHandler := s.Authenticator.Handlers()
 
 	router.Group(func(r chi.Router) {
-		l := logger.New(logger.Log(log.Default()), logger.WithBody, logger.IPfn(ipFn), logger.Prefix("[INFO]"))
-		r.Use(l.Handler, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
+		r.Use(logInfoWithBody, tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)), middleware.NoCache)
 		r.Mount("/auth", authHandler)
 	})
 
@@ -217,10 +214,7 @@ func (s *Rest) routes() chi.Router {
 		// open routes
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(middleware.NoCache)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Use(authMiddleware.Trace, middleware.NoCache, logInfoWithBody)
 			ropen.Get("/find", s.findCommentsCtrl)
 			ropen.Get("/id/{id}", s.commentByIDCtrl)
 			ropen.Get("/comments", s.findUserCommentsCtrl)
@@ -231,44 +225,60 @@ func (s *Rest) routes() chi.Router {
 			ropen.Get("/config", s.configCtrl)
 			ropen.Post("/preview", s.previewCommentCtrl)
 			ropen.Get("/info", s.infoCtrl)
-			ropen.Mount("/img", s.ImageProxy.Routes())
-			ropen.Mount("/rss", s.rssRoutes())
+			ropen.Get("/img", s.ImageProxy.Handler)
+
+			ropen.Route("/rss", func(rrss chi.Router) {
+				rrss.Get("/post", s.rssPostCommentsCtrl)
+				rrss.Get("/site", s.rssSiteCommentsCtrl)
+				rrss.Get("/reply", s.rssRepliesCtrl)
+			})
 		})
 
 		// open routes, cached
 		rapi.Group(func(ropen chi.Router) {
 			ropen.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			ropen.Use(authMiddleware.Trace)
-			ropen.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			ropen.Use(authMiddleware.Trace, logInfoWithBody)
 			ropen.Get("/picture/{user}/{id}", s.loadPictureCtrl)
 		})
 
 		// protected routes, require auth
 		rapi.Group(func(rauth chi.Router) {
 			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
-			rauth.Use(authMiddleware.Auth)
-			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[INFO]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(authMiddleware.Auth, middleware.NoCache, logInfoWithBody)
 			rauth.Get("/user", s.userInfoCtrl)
 			rauth.Get("/userdata", s.userAllDataCtrl)
+		})
 
-			// admin routes, admin users only
-			rauth.Mount("/admin", s.adminService.routes(authMiddleware.AdminOnly))
+		// admin routes, require auth and admin users only
+		rapi.Route("/admin", func(radmin chi.Router) {
+			radmin.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil)))
+			radmin.Use(authMiddleware.Auth, authMiddleware.AdminOnly)
+			radmin.Use(middleware.NoCache, logInfoWithBody)
+
+			radmin.Delete("/comment/{id}", s.adminService.deleteCommentCtrl)
+			radmin.Put("/user/{userid}", s.adminService.setBlockCtrl)
+			radmin.Delete("/user/{userid}", s.adminService.deleteUserCtrl)
+			radmin.Get("/user/{userid}", s.adminService.getUserInfoCtrl)
+			radmin.Get("/deleteme", s.adminService.deleteMeRequestCtrl)
+			radmin.Put("/verify/{userid}", s.adminService.setVerifyCtrl)
+			radmin.Put("/pin/{id}", s.adminService.setPinCtrl)
+			radmin.Get("/blocked", s.adminService.blockedUsersCtrl)
+			radmin.Put("/readonly", s.adminService.setReadOnlyCtrl)
+			radmin.Put("/title/{id}", s.adminService.setTitleCtrl)
+
+			// migrator
+			radmin.Get("/export", s.adminService.migrator.exportCtrl)
+			radmin.Post("/import", s.adminService.migrator.importCtrl)
+			radmin.Post("/import/form", s.adminService.migrator.importFormCtrl)
+			radmin.Get("/import/wait", s.adminService.migrator.importWaitCtrl)
 		})
 
 		// protected routes, throttled to 10/s by default, controlled by external UpdateLimiter param
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
 			rauth.Use(authMiddleware.Auth)
 			rauth.Use(middleware.NoCache)
-			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody,
-				logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
+			rauth.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
 
 			rauth.Put("/comment/{id}", s.updateCommentCtrl)
 			rauth.Post("/comment", s.createCommentCtrl)
@@ -277,39 +287,20 @@ func (s *Rest) routes() chi.Router {
 		})
 
 		rapi.Group(func(rauth chi.Router) {
-			lmt := 10.0
-			if s.UpdateLimiter > 0 {
-				lmt = s.UpdateLimiter
-			}
-			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(lmt, nil)))
-			rauth.Use(authMiddleware.Auth)
+			rauth.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(s.updateLimiter(), nil)))
+			rauth.Use(authMiddleware.Auth, rejectAnonUser)
 			rauth.Use(logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(ipFn)).Handler)
-			rauth.With(rejectAnonUser).Post("/picture", s.savePictureCtrl)
+			rauth.Post("/picture", s.savePictureCtrl)
 		})
 
 	})
 
-	// respond to /robots.txt with the list of allowed paths
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
-				"/img", "/avatar", "/picture"}
-			for i := range allowed {
-				allowed[i] = "Allow: /api/v1" + allowed[i]
-			}
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
-		})
-
-	// respond to /index.html with the content of getstarted.html under /web root
-	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))).
-		Get("/index.html", func(w http.ResponseWriter, r *http.Request) {
-			data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
-			if err != nil {
-				w.WriteHeader(http.StatusNotFound)
-				return
-			}
-			render.HTML(w, r, string(data))
-		})
+	// open routes on root level
+	router.Group(func(rroot chi.Router) {
+		tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))
+		rroot.Get("/index.html", s.getStartedCtrl)
+		rroot.Get("/robots.txt", s.getRobotsCtrl)
+	})
 
 	// file server for static content from /web
 	addFileServer(router, "/web", http.Dir(s.WebRoot))
@@ -351,6 +342,15 @@ func (s *Rest) alterComments(comments []store.Comment, r *http.Request) (res []s
 	return res
 }
 
+// updateLimiter returns UpdateLimiter if set, or 10 if not
+func (s *Rest) updateLimiter() float64 {
+	lmt := 10.0
+	if s.UpdateLimiter > 0 {
+		lmt = s.UpdateLimiter
+	}
+	return lmt
+}
+
 // serves static files from /web or embedded by statik
 func addFileServer(r chi.Router, path string, root http.FileSystem) {
 

backend/app/rest/api/rest_private.go

@@ -9,14 +9,14 @@ import (
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 	"github.com/go-pkgz/auth/token"
 	log "github.com/go-pkgz/lgr"
 	R "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/cache"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"

backend/app/rest/api/rest_public.go

@@ -4,7 +4,9 @@ import (
 	"crypto/sha1" // nolint
 	"encoding/base64"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 
@@ -16,6 +18,7 @@ import (
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 // GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy ]
@@ -38,9 +41,9 @@ func (s *Rest) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 		var b []byte
 		switch r.URL.Query().Get("format") {
 		case "tree":
-			tree := rest.MakeTree(maskedComments, sort, s.ReadOnlyAge)
+			tree := service.MakeTree(maskedComments, sort, s.ReadOnlyAge)
 			if tree.Nodes == nil { // eliminate json nil serialization
-				tree.Nodes = []*rest.Node{}
+				tree.Nodes = []*service.Node{}
 			}
 			if s.DataService.IsReadOnly(locator) {
 				tree.Info.ReadOnly = true
@@ -277,7 +280,7 @@ func (s *Rest) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
 
 	// key could be long for multiple posts, make it sha1
 	k := URLKey(r) + strings.Join(posts, ",")
-	h := sha1.Sum([]byte(k)) //nolint
+	h := sha1.Sum([]byte(k)) // nolint
 	sha := base64.URLEncoding.EncodeToString(h[:])
 
 	key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
@@ -364,7 +367,11 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	defer imgRdr.Close()
+	defer func() {
+		if e := imgRdr.Close(); e != nil {
+			log.Printf("[WARN] failed to close reader for picture %s, %v", id, e)
+		}
+	}()
 
 	w.Header().Set("Content-Type", imgContentType(id))
 	w.Header().Set("Content-Length", strconv.Itoa(int(size)))
@@ -373,3 +380,23 @@ func (s *Rest) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
 		log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
 	}
 }
+
+// GET /index.html - respond to /index.html with the content of getstarted.html under /web root
+func (s *Rest) getStartedCtrl(w http.ResponseWriter, r *http.Request) {
+	data, err := ioutil.ReadFile(path.Join(s.WebRoot, "getstarted.html"))
+	if err != nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	render.HTML(w, r, string(data))
+}
+
+// GET /robots.txt
+func (s *Rest) getRobotsCtrl(w http.ResponseWriter, r *http.Request) {
+	allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config",
+		"/img", "/avatar", "/picture"}
+	for i := range allowed {
+		allowed[i] = "Allow: /api/v1" + allowed[i]
+	}
+	render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")+"\n")
+}

backend/app/rest/api/rest_public_test.go

@@ -14,8 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/store"
+	"github.com/umputun/remark/backend/app/store/service"
 )
 
 func TestRest_Ping(t *testing.T) {
@@ -111,7 +111,7 @@ func TestRest_Find(t *testing.T) {
 	assert.Equal(t, id2, comments.Comments[0].ID)
 
 	// get in tree mode
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -137,7 +137,7 @@ func TestRest_FindAge(t *testing.T) {
 	_, err = srv.DataService.Create(c2)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
@@ -178,7 +178,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	_, err = client.Do(req)
 	require.Nil(t, err)
 
-	tree := rest.Tree{}
+	tree := service.Tree{}
 	res, code := get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah1&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)
@@ -186,7 +186,7 @@ func TestRest_FindReadOnly(t *testing.T) {
 	assert.Equal(t, "https://radio-t.com/blah1", tree.Info.URL)
 	assert.True(t, tree.Info.ReadOnly, "post is ro")
 
-	tree = rest.Tree{}
+	tree = service.Tree{}
 	res, code = get(t, ts.URL+"/api/v1/find?site=radio-t&url=https://radio-t.com/blah2&format=tree")
 	assert.Equal(t, 200, code)
 	err = json.Unmarshal([]byte(res), &tree)

backend/app/rest/api/rest_test.go

@@ -39,7 +39,7 @@ var getStartedHTML = "/tmp/getstarted.html"
 
 var devToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg"
 
-var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.I5a8EHbUJy8mApuYCPDRThbC-1jP0sbPh1qwNyY1V4E"
+var adminUmputunToken = "eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJyYWRpb3QiLCJleHAiOjE5NTQ1OTc5ODAsImp0aSI6Ijk3YTJlMGFjNGRjN2Q1ZjY5MjZkNWU4NjIwYWNlZjlhNDBjMCIsImlhdCI6MTQ1NDU5NzY4MCwiaXNzIjoicmVtYXJrNDIiLCJ1c2VyIjp7Im5hbWUiOiJVbXB1dHVuIiwiaWQiOiJnaXRodWJfZWYwZjcwNmE3IiwicGljdHVyZSI6Imh0dHBzOi8vcmVtYXJrNDIucmFkaW8tdC5jb20vYXBpL3YxL2F2YXRhci9jYjQyZmY0OTNhZGU2OTZkODhhM2E1OTBmMTM2YWU5ZTM0ZGU3YzFiLmltYWdlIiwiYXR0cnMiOnsiYWRtaW4iOnRydWUsImJsb2NrZWQiOmZhbHNlfX19.gAR_sZT7hTx7CNHByyrJQWMB5tAtoiISAiG8kes1IjA"
 
 func TestRest_FileServer(t *testing.T) {
 	ts, _, teardown := startupT(t)
@@ -409,3 +409,13 @@ func addComment(t *testing.T, c store.Comment, ts *httptest.Server) string {
 	time.Sleep(time.Nanosecond * 10)
 	return crResp["id"].(string)
 }
+
+func requireAdminOnly(t *testing.T, req *http.Request) {
+	resp, err := sendReq(t, req, "") // no-auth user
+	require.NoError(t, err)
+	assert.Equal(t, 401, resp.StatusCode)
+
+	resp, err = sendReq(t, req, devToken) // non-admin user
+	require.NoError(t, err)
+	assert.Equal(t, 403, resp.StatusCode)
+}

backend/app/rest/api/rss.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/rest/cache"
 	"github.com/gorilla/feeds"
@@ -22,14 +21,6 @@ const maxReplyDuration = 31 * 24 * time.Hour
 // ui uses links like <post-url>#remark42__comment-<comment-id>
 const uiNav = "#remark42__comment-"
 
-func (s *Rest) rssRoutes() chi.Router {
-	router := chi.NewRouter()
-	router.Get("/post", s.rssPostCommentsCtrl)
-	router.Get("/site", s.rssSiteCommentsCtrl)
-	router.Get("/reply", s.rssRepliesCtrl)
-	return router
-}
-
 // GET /rss/post?site=siteID&url=post-url
 func (s *Rest) rssPostCommentsCtrl(w http.ResponseWriter, r *http.Request) {
 	locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}

backend/app/rest/proxy/image.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/go-chi/chi"
 	log "github.com/go-pkgz/lgr"
 	"github.com/go-pkgz/repeater"
 	"github.com/pkg/errors"
@@ -40,76 +39,75 @@ func (p Image) Convert(commentHTML string) string {
 	return p.replace(commentHTML, imgs)
 }
 
-// Routes returns router group to respond to proxied request
-func (p Image) Routes() chi.Router {
-	router := chi.NewRouter()
+// Handler returns http handler respond to proxied request
+func (p Image) Handler(w http.ResponseWriter, r *http.Request) {
+
 	if !p.Enabled {
-		return router
+		http.Error(w, "proxy disabled", http.StatusNotImplemented)
+		return
 	}
-	router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-		src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
-			return
-		}
 
-		timeout := 60 * time.Second // default
-		if p.Timeout > 0 {
-			timeout = p.Timeout
-		}
+	src, err := base64.URLEncoding.DecodeString(r.URL.Query().Get("src"))
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't decode image url", rest.ErrDecode)
+		return
+	}
 
-		ctx, cancel := context.WithTimeout(context.Background(), timeout)
-		defer cancel()
+	timeout := 60 * time.Second // default
+	if p.Timeout > 0 {
+		timeout = p.Timeout
+	}
 
-		client := http.Client{Timeout: 30 * time.Second}
-		var resp *http.Response
-		err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
-			var e error
-			req, e := http.NewRequest("GET", string(src), nil)
-			if e != nil {
-				return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
-			}
-			resp, e = client.Do(req.WithContext(ctx))
-			return e
-		})
-		if err != nil {
-			rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
-			return
-		}
-		defer func() {
-			if e := resp.Body.Close(); e != nil {
-				log.Printf("[WARN] can't close body, %s", e)
-			}
-		}()
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
 
-		if resp.StatusCode != http.StatusOK {
-			w.WriteHeader(resp.StatusCode)
-			return
-		}
-
-		for k, v := range resp.Header {
-			if strings.EqualFold(k, "Content-Type") {
-				w.Header().Set(k, v[0])
-			}
-			if strings.EqualFold(k, "Content-Length") {
-				w.Header().Set(k, v[0])
-			}
-		}
-		// enforce client-side caching
-		etag := `"` + r.URL.Query().Get("src") + `"`
-		w.Header().Set("Etag", etag)
-		w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
-		if match := r.Header.Get("If-None-Match"); match != "" {
-			if strings.Contains(match, etag) {
-				w.WriteHeader(http.StatusNotModified)
-				return
-			}
-		}
-		if _, e := io.Copy(w, resp.Body); e != nil {
-			log.Printf("[WARN] can't copy image stream, %s", e)
+	client := http.Client{Timeout: 30 * time.Second}
+	var resp *http.Response
+	err = repeater.NewDefault(5, time.Second).Do(ctx, func() error {
+		var e error
+		req, e := http.NewRequest("GET", string(src), nil)
+		if e != nil {
+			return errors.Wrapf(e, "failed to make request for %s", r.URL.Query().Get("src"))
 		}
+		resp, e = client.Do(req.WithContext(ctx))
+		return e
 	})
-	return router
+	if err != nil {
+		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get image "+string(src), rest.ErrAssetNotFound)
+		return
+	}
+	defer func() {
+		if e := resp.Body.Close(); e != nil {
+			log.Printf("[WARN] can't close body, %s", e)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		w.WriteHeader(resp.StatusCode)
+		return
+	}
+
+	for k, v := range resp.Header {
+		if strings.EqualFold(k, "Content-Type") {
+			w.Header().Set(k, v[0])
+		}
+		if strings.EqualFold(k, "Content-Length") {
+			w.Header().Set(k, v[0])
+		}
+	}
+	// enforce client-side caching
+	etag := `"` + r.URL.Query().Get("src") + `"`
+	w.Header().Set("Etag", etag)
+	w.Header().Set("Cache-Control", "max-age=2592000") // 30 days
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if _, e := io.Copy(w, resp.Body); e != nil {
+		log.Printf("[WARN] can't copy image stream, %s", e)
+	}
 }
 
 // extract gets all non-https images and return list of src

backend/app/rest/proxy/image_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -48,9 +49,11 @@ func TestPicture_Extract(t *testing.T) {
 	img := Image{Enabled: true}
 
 	for i, tt := range tbl {
-		res, err := img.extract(tt.inp)
-		assert.Nil(t, err, "err in #%d", i)
-		assert.Equal(t, tt.res, res, "mismatch in #%d", i)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			res, err := img.extract(tt.inp)
+			assert.Nil(t, err)
+			assert.Equal(t, tt.res, res)
+		})
 	}
 }
 
@@ -63,12 +66,11 @@ func TestPicture_Replace(t *testing.T) {
 
 func TestImage_Routes(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy"}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img1.png"))
 
@@ -92,12 +94,11 @@ func TestImage_Routes(t *testing.T) {
 
 func TestImage_RoutesTimedOut(t *testing.T) {
 	img := Image{Enabled: true, RemarkURL: "https://demo.remark42.com", RoutePath: "/api/v1/proxy", Timeout: 50 * time.Millisecond}
-	router := img.Routes()
 
+	ts := httptest.NewServer(http.HandlerFunc(img.Handler))
+	defer ts.Close()
 	httpSrv := imgHTTPServer(t)
 	defer httpSrv.Close()
-	ts := httptest.NewServer(router)
-	defer ts.Close()
 
 	encodedImgURL := base64.URLEncoding.EncodeToString([]byte(httpSrv.URL + "/image/img-slow.png"))
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)

backend/app/store/engine/bolt_accessor.go

@@ -8,7 +8,7 @@ import (
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"

backend/app/store/engine/bolt_accessor_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/bbolt"
+	bolt "github.com/coreos/bbolt"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 

backend/app/store/engine/mongo.go

@@ -6,7 +6,7 @@ import (
 	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"github.com/go-pkgz/mongo"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
 	"github.com/umputun/remark/backend/app/store"
@@ -39,7 +39,7 @@ type metaUser struct {
 }
 
 // NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
-// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
+// flushDuration triggers automatic flush (write from buffer), 0 disables it and will flush as buffer size reached.
 // important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
 func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
 	writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)

backend/app/store/service/service.go

@@ -9,14 +9,14 @@ import (
 
 	log "github.com/go-pkgz/lgr"
 	"github.com/google/uuid"
-	multierror "github.com/hashicorp/go-multierror"
-	cache "github.com/patrickmn/go-cache"
+	"github.com/hashicorp/go-multierror"
+	"github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
-	"github.com/umputun/remark/backend/app/store/image"
 
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
+	"github.com/umputun/remark/backend/app/store/image"
 )
 
 // DataStore wraps store.Interface with additional methods
@@ -207,8 +207,8 @@ func (s *DataStore) Vote(locator store.Locator, commentID string, userID string,
 	}
 
 	comment.Vote = 0
-	if v, ok := comment.Votes[userID]; ok {
-		if v {
+	if vv, ok := comment.Votes[userID]; ok {
+		if vv {
 			comment.Vote = 1
 		} else {
 			comment.Vote = -1

backend/app/store/service/title.go

@@ -44,7 +44,11 @@ func (t *TitleExtractor) Get(url string) (string, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to load page %s", url)
 		}
-		defer resp.Body.Close() //nolint
+		defer func() {
+			if err = resp.Body.Close(); err != nil {
+				log.Printf("[WARN] failed to close title extractor body, %v", err)
+			}
+		}()
 		if resp.StatusCode != 200 {
 			return nil, errors.Errorf("can't load page %s, code %d", url, resp.StatusCode)
 		}

backend/app/store/service/tree.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"sort"

backend/app/store/service/tree_test.go

@@ -1,4 +1,4 @@
-package rest
+package service
 
 import (
 	"encoding/json"

backend/app/store/user.go

@@ -2,7 +2,7 @@ package store
 
 import (
 	"crypto/hmac"
-	"crypto/sha1" //nolint
+	"crypto/sha1" // nolint
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -41,7 +41,7 @@ func HashValue(val string, secret string) string {
 // EncodeID hashes id to sha1. The function intentionally left outside of User struct because in some cases
 // we need hashing for parts of id, in some others hashing for non-User values.
 func EncodeID(id string) string {
-	return hashWithFallback(sha1.New(), id) //nolint
+	return hashWithFallback(sha1.New(), id) // nolint
 }
 
 // hashWithFallback tries to has val with hash.Hash and fallback to crc if needed

backend/go.mod

@@ -3,9 +3,10 @@ module github.com/umputun/remark/backend
 go 1.12
 
 require (
+	cloud.google.com/go v0.38.0 // indirect
 	github.com/PuerkitoBio/goquery v1.4.0
 	github.com/andybalholm/cascadia v1.0.0 // indirect
-	github.com/coreos/bbolt v1.3.0
+	github.com/coreos/bbolt v1.3.2
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/didip/tollbooth v4.0.0+incompatible
 	github.com/didip/tollbooth_chi v0.0.0-20170928041846-6ab5f3083f3d
@@ -13,18 +14,19 @@ require (
 	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/go-chi/cors v1.0.0
 	github.com/go-chi/render v1.0.0
-	github.com/go-pkgz/auth v0.5.0
+	github.com/go-pkgz/auth v0.5.2
 	github.com/go-pkgz/lcw v0.2.0
 	github.com/go-pkgz/lgr v0.6.2
 	github.com/go-pkgz/mongo v1.1.2
 	github.com/go-pkgz/repeater v1.1.1
-	github.com/go-pkgz/rest v1.4.0
+	github.com/go-pkgz/rest v1.4.1
 	github.com/go-pkgz/syncs v1.1.0
-	github.com/golang/mock v1.2.0
+	github.com/golang/protobuf v1.3.1 // indirect
 	github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c
 	github.com/gorilla/feeds v1.1.0
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc
 	github.com/microcosm-cc/bluemonday v0.0.0-20171222152607-542fd4642604
 	github.com/patrickmn/go-cache v2.1.0+incompatible
@@ -32,10 +34,16 @@ require (
 	github.com/rakyll/statik v0.1.3
 	github.com/rs/xid v1.2.1
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.3.0
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b
-	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2
+	go.etcd.io/bbolt v1.3.2 // indirect
+	golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529
+	golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec
+	golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect
+	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
 	gopkg.in/russross/blackfriday.v2 v2.0.0
 )

backend/go.sum

@@ -1,13 +1,20 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/PuerkitoBio/goquery v1.4.0 h1:13fV4AYmaSopdNp8KWDUlLyU5INklBkYk0tsTfxRO2U=
 github.com/PuerkitoBio/goquery v1.4.0/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
 github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.0 h1:HIgH5xUWXT914HCI671AxuTTqjj64UOFr7pHn48LUTI=
 github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -26,8 +33,8 @@ github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0=
 github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-chi/render v1.0.0 h1:cLJlkaTB4xfx5rWhtoB0BSXsXVJKWFqv08Y3cR1bZKA=
 github.com/go-chi/render v1.0.0/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
-github.com/go-pkgz/auth v0.5.0 h1:+wqppq35x83PchZNZ7SHHYLI/e8WeETFouujDLsklac=
-github.com/go-pkgz/auth v0.5.0/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
+github.com/go-pkgz/auth v0.5.2 h1:Sdu2K6iZxMDt5nQSAomEbzJhmIoUqNwfFMiJwYV8Xn4=
+github.com/go-pkgz/auth v0.5.2/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
 github.com/go-pkgz/lcw v0.2.0 h1:aFoKUG8q0YybId+ThVRQpDMjjuSG4hkLL1EA2xUtruc=
 github.com/go-pkgz/lcw v0.2.0/go.mod h1:k+PY1CkCMTLXILtFoJOyK65Qqi9rkoTYunFH1vE/C0I=
 github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
@@ -39,16 +46,25 @@ github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdP
 github.com/go-pkgz/repeater v1.1.1 h1:9HVgXFJGjUQznPmaeuVDTPhgflzVlUyjCx2gmBYXeGI=
 github.com/go-pkgz/repeater v1.1.1/go.mod h1:QfNR/a+xqjs+f9wSxWqOQlw9aQhmKlUaSwXCiZ+Ko2w=
 github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
-github.com/go-pkgz/rest v1.4.0 h1:xNkdMjEL2rNZSHouWjFTH22ncaZ77fopm34RN+eXAwk=
-github.com/go-pkgz/rest v1.4.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
+github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
+github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
 github.com/go-pkgz/syncs v1.1.0 h1:k+dTyUZs1JHsYzo2tuUNrnW0OCwuGuS6ozfXHVspjSY=
 github.com/go-pkgz/syncs v1.1.0/go.mod h1:bt9lxWRRJ9vOCMGc8Big8ttjYHLKP88ofj1y38UlaHE=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c=
 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gorilla/feeds v1.1.0 h1:pcgLJhbdYgaUESnj3AmXPcB7cS3vy63+jC/TI14AGXk=
 github.com/gorilla/feeds v1.1.0/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -57,8 +73,11 @@ github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnz
 github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc h1:0L2sGkaj6MWuV1BfXsrLJ/+XA8RzKKVsYlLVXNkK1Lw=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -84,34 +103,80 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/umputun/remark v1.2.0 h1:RoKBgzjow7+t4Z1XbhCIOiLcZNuE6LGuvj+Gxh4mopI=
+go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec h1:arXJwtMuk5vqI1NHX0UTnNw977rYk5Sl4jQqHj+hun4=
+golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8=
 golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 h1:6M3SDHlHHDCx2PcQw3S4KsR170vGqDhJDOmpVd4Hjak=
+golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
+golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
 gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

backend/remark.rest

@@ -5,7 +5,7 @@ GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=tree&url={{url}}
 ### find request with plain
 GET {{host}}/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
 
-### find request with plain
+### find request with plain. dev token for secret=secret, not admin
 GET http://127.0.0.1:8080/api/v1/find?site={{site}}&sort=-controversy&format=plain&url={{url}}
 X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
 
@@ -112,7 +112,7 @@ GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
 ### site rss
 GET {{host}}/api/v1/rss/site?site={{site}}
 
-### replie rss
+### reply rss
 GET {{host}}/api/v1/rss/reply?site={{site}}&user={{user}}
 
 ### get default avatar
@@ -121,5 +121,14 @@ GET {{host}}/api/v1/avatar/blah
 ### get config
 GET  {{host}}/api/v1/config?site={{site}}
 
+### deleteme (use request). dev token for secret=secret, not admin
+POST {{host}}/api/v1/deleteme?site_id={{site}}
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcms0MiIsImV4cCI6Mzc4OTE5MTgyMiwianRpIjoicmFuZG9tIGlkIiwiaXNzIjoicmVtYXJrNDIiLCJuYmYiOjE1MjE4ODQyMjIsInVzZXIiOnsibmFtZSI6ImRldmVsb3BlciBvbmUiLCJpZCI6ImRldiIsInBpY3R1cmUiOiJodHRwOi8vZXhhbXBsZS5jb20vcGljLnBuZyIsImlwIjoiMTI3LjAuMC4xIiwiZW1haWwiOiJtZUBleGFtcGxlLmNvbSJ9fQ.aKUAXiZxXypgV7m1wEOgUcyPOvUDXHDi3A06YWKbcLg
+
+### deletme, admin. admin token for secret=secret
+GET {{host}}/api/v1/admin/deleteme?token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1NjU2NjI4MDYsImlzcyI6InJlbWFyazQyIiwibmJmIjoxNTU3NzEzOTQ2LCJ1c2VyIjp7Im5hbWUiOiJkZXZlbG9wZXIgb25lIiwiaWQiOiJkZXYiLCJwaWN0dXJlIjoiIiwiYXR0cnMiOnsiYWRtaW4iOmZhbHNlLCJibG9ja2VkIjpmYWxzZSwiZGVsZXRlX21lIjp0cnVlfX19.qmnsQt_jilHzoauA9D7t1m3w69qvAJsZkuKVPWhpdik
+X-JWT: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJyZW1hcmsiLCJleHAiOjE2NTc3MDQ5NzksImp0aSI6Ijk4ZDNhOGFkMGNjZWU5M2Q1MWYwYjJiOTY1ZjU2YmE2NmJkNmZiNzYiLCJpYXQiOjE1NTc3MDQ2NzksImlzcyI6InJlbWFyazQyIiwidXNlciI6eyJuYW1lIjoiZGV2X3VzZXIiLCJpZCI6ImRldl91c2VyIiwicGljdHVyZSI6Imh0dHA6Ly8xMjcuMC4wLjE6ODA4MC9hcGkvdjEvYXZhdGFyL2NjZmEyYWJkMDE2Njc2MDViNGUxZmM0ZmNiOTFiMWUxYWYzMjMyNDAuaW1hZ2UiLCJhdHRycyI6eyJhZG1pbiI6dHJ1ZSwiYmxvY2tlZCI6ZmFsc2V9fX0.YIt9Zq3n0O8PMkR78pGAqyHI0exCU0vLKjcHgcXfLFw
+
+
 ### ping
 GET {{host}}/ping

backend/vendor/cloud.google.com/go/AUTHORS

@@ -1,15 +0,0 @@
-# This is the official list of cloud authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as:
-# Name or Organization <email address>
-# The email address is not required for organizations.
-
-Filippo Valsorda <hi@filippo.io>
-Google Inc.
-Ingo Oeser <nightlyone@googlemail.com>
-Palm Stone Games, Inc.
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/CONTRIBUTORS

@@ -1,40 +0,0 @@
-# People who have agreed to one of the CLAs and can contribute patches.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# https://developers.google.com/open-source/cla/individual
-# https://developers.google.com/open-source/cla/corporate
-#
-# Names should be added to this file as:
-#     Name <email address>
-
-# Keep the list alphabetically sorted.
-
-Alexis Hunt <lexer@google.com>
-Andreas Litt <andreas.litt@gmail.com>
-Andrew Gerrand <adg@golang.org>
-Brad Fitzpatrick <bradfitz@golang.org>
-Burcu Dogan <jbd@google.com>
-Dave Day <djd@golang.org>
-David Sansome <me@davidsansome.com>
-David Symonds <dsymonds@golang.org>
-Filippo Valsorda <hi@filippo.io>
-Glenn Lewis <gmlewis@google.com>
-Ingo Oeser <nightlyone@googlemail.com>
-James Hall <james.hall@shopify.com>
-Johan Euphrosine <proppy@google.com>
-Jonathan Amsterdam <jba@google.com>
-Kunpei Sakai <namusyaka@gmail.com>
-Luna Duclos <luna.duclos@palmstonegames.com>
-Magnus Hiie <magnus.hiie@gmail.com>
-Mario Castro <mariocaster@gmail.com>
-Michael McGreevy <mcgreevy@golang.org>
-Omar Jarjur <ojarjur@google.com>
-Paweł Knap <pawelknap88@gmail.com>
-Péter Szilágyi <peterke@gmail.com>
-Sarah Adams <shadams@google.com>
-Thanatat Tamtan <acoshift@gmail.com>
-Toby Burress <kurin@google.com>
-Tuo Shan <shantuo@google.com>
-Tyler Treat <ttreat31@gmail.com>

backend/vendor/cloud.google.com/go/compute/metadata/metadata.go

@@ -137,7 +137,7 @@ func testOnGCE() bool {
 	resc := make(chan bool, 2)
 
 	// Try two strategies in parallel.
-	// See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194
+	// See https://github.com/googleapis/google-cloud-go/issues/194
 	go func() {
 		req, _ := http.NewRequest("GET", "http://"+metadataIP, nil)
 		req.Header.Set("User-Agent", userAgent)
@@ -300,8 +300,8 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 		// being stable anyway.
 		host = metadataIP
 	}
-	url := "http://" + host + "/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
+	u := "http://" + host + "/computeMetadata/v1/" + suffix
+	req, _ := http.NewRequest("GET", u, nil)
 	req.Header.Set("Metadata-Flavor", "Google")
 	req.Header.Set("User-Agent", userAgent)
 	res, err := c.hc.Do(req)
@@ -312,13 +312,13 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 	if res.StatusCode == http.StatusNotFound {
 		return "", "", NotDefinedError(suffix)
 	}
-	if res.StatusCode != 200 {
-		return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
 	all, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", "", err
 	}
+	if res.StatusCode != 200 {
+		return "", "", &Error{Code: res.StatusCode, Message: string(all)}
+	}
 	return string(all), res.Header.Get("Etag"), nil
 }
 
@@ -499,3 +499,15 @@ func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) erro
 		}
 	}
 }
+
+// Error contains an error response from the server.
+type Error struct {
+	// Code is the HTTP response status code.
+	Code int
+	// Message is the server response message.
+	Message string
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("compute: Received %d `%s`", e.Code, e.Message)
+}

backend/vendor/github.com/coreos/bbolt/.gitignore

@@ -2,3 +2,4 @@
 *.test
 *.swp
 /bin/
+cover.out

backend/vendor/github.com/coreos/bbolt/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+go_import_path: go.etcd.io/bbolt
+
+sudo: false
+
+go:
+- 1.11
+
+before_install:
+- go get -v honnef.co/go/tools/...
+- go get -v github.com/kisielk/errcheck
+
+script:
+- make fmt
+- make test
+- make race
+# - make errcheck

backend/vendor/github.com/coreos/bbolt/Makefile

@@ -5,14 +5,34 @@ GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 default: build
 
 race:
-	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@TEST_FREELIST_TYPE=hashmap go test -v -race -test.run="TestSimulate_(100op|1000op)"
+	@echo "array freelist test"
+	@TEST_FREELIST_TYPE=array go test -v -race -test.run="TestSimulate_(100op|1000op)"
+
+fmt:
+	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
+
+# go get honnef.co/go/tools/simple
+gosimple:
+	gosimple ./...
+
+# go get honnef.co/go/tools/unused
+unused:
+	unused ./...
 
 # go get github.com/kisielk/errcheck
 errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/boltdb/bolt
+	@errcheck -ignorepkg=bytes -ignore=os:Remove go.etcd.io/bbolt
 
-test: 
-	@go test -v -cover .
-	@go test -v ./cmd/bolt
+test:
+	TEST_FREELIST_TYPE=hashmap go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	TEST_FREELIST_TYPE=hashmap go test -v ./cmd/bbolt
 
-.PHONY: fmt test
+	@echo "array freelist test"
+
+	@TEST_FREELIST_TYPE=array go test -timeout 20m -v -coverprofile cover.out -covermode atomic
+	# Note: gets "program not an importable package" in out of path builds
+	@TEST_FREELIST_TYPE=array go test -v ./cmd/bbolt
+
+.PHONY: race fmt errcheck test gosimple unused

backend/vendor/github.com/coreos/bbolt/README.md

@@ -1,5 +1,18 @@
-Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg)
-====
+bbolt
+=====
+
+[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt)
+[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt)
+[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt)
+[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt)
+[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases)
+[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE)
+
+bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value
+store. The purpose of this fork is to provide the Go community with an active
+maintenance and development target for Bolt; the goal is improved reliability
+and stability. bbolt includes bug fixes, performance enhancements, and features
+not found in Bolt while preserving backwards compatibility with the Bolt API.
 
 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
 [LMDB project][lmdb]. The goal of the project is to provide a simple,
@@ -10,47 +23,55 @@ Since Bolt is meant to be used as such a low-level piece of functionality,
 simplicity is key. The API will be small and only focus on getting values
 and setting values. That's it.
 
+[gh_ben]: https://github.com/benbjohnson
+[bolt]: https://github.com/boltdb/bolt
 [hyc_symas]: https://twitter.com/hyc_symas
 [lmdb]: http://symas.com/mdb/
 
 ## Project Status
 
-Bolt is stable and the API is fixed. Full unit test coverage and randomized
-black box testing are used to ensure database consistency and thread safety.
-Bolt is currently in high-load production environments serving databases as
-large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed
-services every day.
+Bolt is stable, the API is fixed, and the file format is fixed. Full unit
+test coverage and randomized black box testing are used to ensure database
+consistency and thread safety. Bolt is currently used in high-load production
+environments serving databases as large as 1TB. Many companies such as
+Shopify and Heroku use Bolt-backed services every day.
+
+## Project versioning
+
+bbolt uses [semantic versioning](http://semver.org).
+API should not change between patch and minor releases.
+New minor versions may add additional features to the API.
 
 ## Table of Contents
 
-- [Getting Started](#getting-started)
-  - [Installing](#installing)
-  - [Opening a database](#opening-a-database)
-  - [Transactions](#transactions)
-    - [Read-write transactions](#read-write-transactions)
-    - [Read-only transactions](#read-only-transactions)
-    - [Batch read-write transactions](#batch-read-write-transactions)
-    - [Managing transactions manually](#managing-transactions-manually)
-  - [Using buckets](#using-buckets)
-  - [Using key/value pairs](#using-keyvalue-pairs)
-  - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
-  - [Iterating over keys](#iterating-over-keys)
-    - [Prefix scans](#prefix-scans)
-    - [Range scans](#range-scans)
-    - [ForEach()](#foreach)
-  - [Nested buckets](#nested-buckets)
-  - [Database backups](#database-backups)
-  - [Statistics](#statistics)
-  - [Read-Only Mode](#read-only-mode)
-  - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
-- [Resources](#resources)
-- [Comparison with other databases](#comparison-with-other-databases)
-  - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
-  - [LevelDB, RocksDB](#leveldb-rocksdb)
-  - [LMDB](#lmdb)
-- [Caveats & Limitations](#caveats--limitations)
-- [Reading the Source](#reading-the-source)
-- [Other Projects Using Bolt](#other-projects-using-bolt)
+  - [Getting Started](#getting-started)
+    - [Installing](#installing)
+    - [Opening a database](#opening-a-database)
+    - [Transactions](#transactions)
+      - [Read-write transactions](#read-write-transactions)
+      - [Read-only transactions](#read-only-transactions)
+      - [Batch read-write transactions](#batch-read-write-transactions)
+      - [Managing transactions manually](#managing-transactions-manually)
+    - [Using buckets](#using-buckets)
+    - [Using key/value pairs](#using-keyvalue-pairs)
+    - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
+    - [Iterating over keys](#iterating-over-keys)
+      - [Prefix scans](#prefix-scans)
+      - [Range scans](#range-scans)
+      - [ForEach()](#foreach)
+    - [Nested buckets](#nested-buckets)
+    - [Database backups](#database-backups)
+    - [Statistics](#statistics)
+    - [Read-Only Mode](#read-only-mode)
+    - [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
+  - [Resources](#resources)
+  - [Comparison with other databases](#comparison-with-other-databases)
+    - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
+    - [LevelDB, RocksDB](#leveldb-rocksdb)
+    - [LMDB](#lmdb)
+  - [Caveats & Limitations](#caveats--limitations)
+  - [Reading the Source](#reading-the-source)
+  - [Other Projects Using Bolt](#other-projects-using-bolt)
 
 ## Getting Started
 
@@ -59,13 +80,28 @@ services every day.
 To start using Bolt, install Go and run `go get`:
 
 ```sh
-$ go get github.com/boltdb/bolt/...
+$ go get go.etcd.io/bbolt/...
 ```
 
 This will retrieve the library and install the `bolt` command line utility into
 your `$GOBIN` path.
 
 
+### Importing bbolt
+
+To use bbolt as an embedded key-value store, import as:
+
+```go
+import bolt "go.etcd.io/bbolt"
+
+db, err := bolt.Open(path, 0666, nil)
+if err != nil {
+  return err
+}
+defer db.Close()
+```
+
+
 ### Opening a database
 
 The top-level object in Bolt is a `DB`. It is represented as a single file on
@@ -79,7 +115,7 @@ package main
 import (
 	"log"
 
-	"github.com/boltdb/bolt"
+	bolt "go.etcd.io/bbolt"
 )
 
 func main() {
@@ -209,7 +245,7 @@ and then safely close your transaction if an error is returned. This is the
 recommended way to use Bolt transactions.
 
 However, sometimes you may want to manually start and end your transactions.
-You can use the `Tx.Begin()` function directly but **please** be sure to close
+You can use the `DB.Begin()` function directly but **please** be sure to close
 the transaction.
 
 ```go
@@ -395,7 +431,7 @@ db.View(func(tx *bolt.Tx) error {
 	c := tx.Bucket([]byte("MyBucket")).Cursor()
 
 	prefix := []byte("1234")
-	for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
 		fmt.Printf("key=%s, value=%s\n", k, v)
 	}
 
@@ -448,6 +484,10 @@ db.View(func(tx *bolt.Tx) error {
 })
 ```
 
+Please note that keys and values in `ForEach()` are only valid while
+the transaction is open. If you need to use a key or value outside of
+the transaction, you must use `copy()` to copy it to another byte
+slice.
 
 ### Nested buckets
 
@@ -460,6 +500,55 @@ func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error)
 func (*Bucket) DeleteBucket(key []byte) error
 ```
 
+Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings.
+
+```go
+
+// createUser creates a new user in the given account.
+func createUser(accountID int, u *User) error {
+    // Start the transaction.
+    tx, err := db.Begin(true)
+    if err != nil {
+        return err
+    }
+    defer tx.Rollback()
+
+    // Retrieve the root bucket for the account.
+    // Assume this has already been created when the account was set up.
+    root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10)))
+
+    // Setup the users bucket.
+    bkt, err := root.CreateBucketIfNotExists([]byte("USERS"))
+    if err != nil {
+        return err
+    }
+
+    // Generate an ID for the new user.
+    userID, err := bkt.NextSequence()
+    if err != nil {
+        return err
+    }
+    u.ID = userID
+
+    // Marshal and save the encoded user.
+    if buf, err := json.Marshal(u); err != nil {
+        return err
+    } else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil {
+        return err
+    }
+
+    // Commit the transaction.
+    if err := tx.Commit(); err != nil {
+        return err
+    }
+
+    return nil
+}
+
+```
+
+
+
 
 ### Database backups
 
@@ -469,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block
 your other database reads and writes.
 
 By default, it will use a regular file handle which will utilize the operating
-system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx)
+system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx)
 documentation for information about optimizing for larger-than-RAM datasets.
 
 One common use case is to backup over HTTP so you can use tools like `cURL` to
@@ -715,6 +804,9 @@ Here are a few things to note when evaluating and using Bolt:
   can be reused by a new page or can be unmapped from virtual memory and you'll
   see an `unexpected fault address` panic when accessing it.
 
+* Bolt uses an exclusive write lock on the database file so it cannot be
+  shared by multiple processes.
+
 * Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
   buckets that have random inserts will cause your database to have very poor
   page utilization.
@@ -755,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt:
 
 ## Reading the Source
 
-Bolt is a relatively small code base (<3KLOC) for an embedded, serializable,
+Bolt is a relatively small code base (<5KLOC) for an embedded, serializable,
 transactional key/value database so it can be a good starting point for people
 interested in how databases work.
 
@@ -807,46 +899,56 @@ them via pull request.
 
 Below is a list of public, open source projects that use Bolt:
 
-* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
-* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
+* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
 * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
-* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
-* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
-* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
-* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
-* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
-* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
-* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
-* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
-* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
+* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal.
+* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support.
+* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB
 * [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
-* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
-* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
-* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
-* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
+* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
+* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
 * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
-* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
-* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
-* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
-* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
-* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
-* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
-* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
-* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
-* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
-  backed by boltdb.
+* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet.
 * [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining
   simple tx and key scans.
-* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
-* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
+* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
+* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
+* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
+* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency.
+* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
+* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
+* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
 * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
-* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
-* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
-* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
 * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
-* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
-* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
-* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
 * [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter.
+* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains
+* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
+* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
+* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
+* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
+* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies
+* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
+* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding.
+* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
+* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
+* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
+* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
+* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
+* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
+* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
+* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
+* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi.
+* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
+* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
+* [stow](https://github.com/djherbis/stow) -  a persistence manager for objects
+  backed by boltdb.
+* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
+* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
+* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
+* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
+* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
+* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development.
+* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
 
 If you are using Bolt in a project please send a pull request to add it to the list.

backend/vendor/github.com/coreos/bbolt/appveyor.yml

@@ -1,18 +0,0 @@
-version: "{build}"
-
-os: Windows Server 2012 R2
-
-clone_folder: c:\gopath\src\github.com\boltdb\bolt
-
-environment:
-  GOPATH: c:\gopath
-
-install:
-  - echo %PATH%
-  - echo %GOPATH%
-  - go version
-  - go env
-  - go get -v -t ./...
-
-build_script:
-  - go test -v ./...

backend/vendor/github.com/coreos/bbolt/bolt_386.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_amd64.go

@@ -1,7 +1,10 @@
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_arm.go

@@ -1,7 +1,28 @@
-package bolt
+package bbolt
+
+import "unsafe"
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned bool
+
+func init() {
+	// Simple check to see whether this arch handles unaligned load/stores
+	// correctly.
+
+	// ARM9 and older devices require load/stores to be from/to aligned
+	// addresses. If not, the lower 2 bits are cleared and that address is
+	// read in a jumbled up order.
+
+	// See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+
+	raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11}
+	val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2))
+
+	brokenUnaligned = val != 0x11222211
+}

backend/vendor/github.com/coreos/bbolt/bolt_arm64.go

@@ -1,9 +1,12 @@
 // +build arm64
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_linux.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"syscall"

backend/vendor/github.com/coreos/bbolt/bolt_mips64x.go

@@ -0,0 +1,12 @@
+// +build mips64 mips64le
+
+package bbolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x8000000000 // 512GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_mipsx.go

@@ -0,0 +1,12 @@
+// +build mips mipsle
+
+package bbolt
+
+// maxMapSize represents the largest mmap size supported by Bolt.
+const maxMapSize = 0x40000000 // 1GB
+
+// maxAllocSize is the size used when creating array pointers.
+const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_openbsd.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"syscall"

backend/vendor/github.com/coreos/bbolt/bolt_ppc.go

@@ -1,9 +1,12 @@
 // +build ppc
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0x7FFFFFFF // 2GB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0xFFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_ppc64.go

@@ -1,9 +1,12 @@
 // +build ppc64
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_ppc64le.go

@@ -1,9 +1,12 @@
 // +build ppc64le
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_s390x.go

@@ -1,9 +1,12 @@
 // +build s390x
 
-package bolt
+package bbolt
 
 // maxMapSize represents the largest mmap size supported by Bolt.
 const maxMapSize = 0xFFFFFFFFFFFF // 256TB
 
 // maxAllocSize is the size used when creating array pointers.
 const maxAllocSize = 0x7FFFFFFF
+
+// Are unaligned load/stores broken on this arch?
+var brokenUnaligned = false

backend/vendor/github.com/coreos/bbolt/bolt_unix.go

@@ -1,41 +1,43 @@
 // +build !windows,!plan9,!solaris
 
-package bolt
+package bbolt
 
 import (
 	"fmt"
-	"os"
 	"syscall"
 	"time"
 	"unsafe"
 )
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	fd := db.file.Fd()
+	flag := syscall.LOCK_NB
+	if exclusive {
+		flag |= syscall.LOCK_EX
+	} else {
+		flag |= syscall.LOCK_SH
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		flag := syscall.LOCK_SH
-		if exclusive {
-			flag = syscall.LOCK_EX
-		}
-
-		// Otherwise attempt to obtain an exclusive lock.
-		err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB)
+		// Attempt to obtain an exclusive lock.
+		err := syscall.Flock(int(fd), flag)
 		if err == nil {
 			return nil
 		} else if err != syscall.EWOULDBLOCK {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
@@ -53,7 +55,9 @@ func mmap(db *DB, sz int) error {
 	}
 
 	// Advise the kernel that the mmap is accessed randomly.
-	if err := madvise(b, syscall.MADV_RANDOM); err != nil {
+	err = madvise(b, syscall.MADV_RANDOM)
+	if err != nil && err != syscall.ENOSYS {
+		// Ignore not implemented error in kernel because it still works.
 		return fmt.Errorf("madvise: %s", err)
 	}
 

backend/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go

@@ -1,8 +1,7 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
-	"os"
 	"syscall"
 	"time"
 	"unsafe"
@@ -11,36 +10,35 @@ import (
 )
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	fd := db.file.Fd()
+	var lockType int16
+	if exclusive {
+		lockType = syscall.F_WRLCK
+	} else {
+		lockType = syscall.F_RDLCK
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
-		var lock syscall.Flock_t
-		lock.Start = 0
-		lock.Len = 0
-		lock.Pid = 0
-		lock.Whence = 0
-		lock.Pid = 0
-		if exclusive {
-			lock.Type = syscall.F_WRLCK
-		} else {
-			lock.Type = syscall.F_RDLCK
-		}
-		err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock)
+		// Attempt to obtain an exclusive lock.
+		lock := syscall.Flock_t{Type: lockType}
+		err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock)
 		if err == nil {
 			return nil
 		} else if err != syscall.EAGAIN {
 			return err
 		}
 
+		// If we timed out then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 

backend/vendor/github.com/coreos/bbolt/bolt_windows.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -16,8 +16,6 @@ var (
 )
 
 const (
-	lockExt = ".lock"
-
 	// see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
 	flagLockExclusive       = 2
 	flagLockFailImmediately = 1
@@ -48,48 +46,47 @@ func fdatasync(db *DB) error {
 }
 
 // flock acquires an advisory lock on a file descriptor.
-func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
-	// Create a separate lock file on windows because a process
-	// cannot share an exclusive lock on the same file. This is
-	// needed during Tx.WriteTo().
-	f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode)
-	if err != nil {
-		return err
-	}
-	db.lockfile = f
-
+func flock(db *DB, exclusive bool, timeout time.Duration) error {
 	var t time.Time
+	if timeout != 0 {
+		t = time.Now()
+	}
+	var flag uint32 = flagLockFailImmediately
+	if exclusive {
+		flag |= flagLockExclusive
+	}
 	for {
-		// If we're beyond our timeout then return an error.
-		// This can only occur after we've attempted a flock once.
-		if t.IsZero() {
-			t = time.Now()
-		} else if timeout > 0 && time.Since(t) > timeout {
-			return ErrTimeout
-		}
+		// Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range
+		// -1..0 as the lock on the database file.
+		var m1 uint32 = (1 << 32) - 1 // -1 in a uint32
+		err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{
+			Offset:     m1,
+			OffsetHigh: m1,
+		})
 
-		var flag uint32 = flagLockFailImmediately
-		if exclusive {
-			flag |= flagLockExclusive
-		}
-
-		err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{})
 		if err == nil {
 			return nil
 		} else if err != errLockViolation {
 			return err
 		}
 
+		// If we timed oumercit then return an error.
+		if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout {
+			return ErrTimeout
+		}
+
 		// Wait for a bit and try again.
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(flockRetryTimeout)
 	}
 }
 
 // funlock releases an advisory lock on a file descriptor.
 func funlock(db *DB) error {
-	err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{})
-	db.lockfile.Close()
-	os.Remove(db.path+lockExt)
+	var m1 uint32 = (1 << 32) - 1 // -1 in a uint32
+	err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{
+		Offset:     m1,
+		OffsetHigh: m1,
+	})
 	return err
 }
 

backend/vendor/github.com/coreos/bbolt/boltsync_unix.go

@@ -1,6 +1,6 @@
 // +build !windows,!plan9,!linux,!openbsd
 
-package bolt
+package bbolt
 
 // fdatasync flushes written data to a file descriptor.
 func fdatasync(db *DB) error {

backend/vendor/github.com/coreos/bbolt/bucket.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"bytes"
@@ -14,13 +14,6 @@ const (
 	MaxValueSize = (1 << 31) - 2
 )
 
-const (
-	maxUint = ^uint(0)
-	minUint = 0
-	maxInt  = int(^uint(0) >> 1)
-	minInt  = -maxInt - 1
-)
-
 const bucketHeaderSize = int(unsafe.Sizeof(bucket{}))
 
 const (
@@ -130,9 +123,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket {
 func (b *Bucket) openBucket(value []byte) *Bucket {
 	var child = newBucket(b.tx)
 
+	// If unaligned load/stores are broken on this arch and value is
+	// unaligned simply clone to an aligned byte array.
+	unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0
+
+	if unaligned {
+		value = cloneBytes(value)
+	}
+
 	// If this is a writable transaction then we need to copy the bucket entry.
 	// Read-only transactions can point directly at the mmap entry.
-	if b.tx.writable {
+	if b.tx.writable && !unaligned {
 		child.bucket = &bucket{}
 		*child.bucket = *(*bucket)(unsafe.Pointer(&value[0]))
 	} else {
@@ -167,9 +168,8 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) {
 	if bytes.Equal(key, k) {
 		if (flags & bucketLeafFlag) != 0 {
 			return nil, ErrBucketExists
-		} else {
-			return nil, ErrIncompatibleValue
 		}
+		return nil, ErrIncompatibleValue
 	}
 
 	// Create empty, inline bucket.
@@ -316,7 +316,12 @@ func (b *Bucket) Delete(key []byte) error {
 
 	// Move cursor to correct position.
 	c := b.Cursor()
-	_, _, flags := c.seek(key)
+	k, _, flags := c.seek(key)
+
+	// Return nil if the key doesn't exist.
+	if !bytes.Equal(key, k) {
+		return nil
+	}
 
 	// Return an error if there is already existing bucket value.
 	if (flags & bucketLeafFlag) != 0 {
@@ -329,6 +334,28 @@ func (b *Bucket) Delete(key []byte) error {
 	return nil
 }
 
+// Sequence returns the current integer for the bucket without incrementing it.
+func (b *Bucket) Sequence() uint64 { return b.bucket.sequence }
+
+// SetSequence updates the sequence number for the bucket.
+func (b *Bucket) SetSequence(v uint64) error {
+	if b.tx.db == nil {
+		return ErrTxClosed
+	} else if !b.Writable() {
+		return ErrTxNotWritable
+	}
+
+	// Materialize the root node if it hasn't been already so that the
+	// bucket will be saved during commit.
+	if b.rootNode == nil {
+		_ = b.node(b.root, nil)
+	}
+
+	// Increment and return the sequence.
+	b.bucket.sequence = v
+	return nil
+}
+
 // NextSequence returns an autoincrementing integer for the bucket.
 func (b *Bucket) NextSequence() (uint64, error) {
 	if b.tx.db == nil {

backend/vendor/github.com/coreos/bbolt/cursor.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"bytes"
@@ -157,12 +157,6 @@ func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) {
 	// Start from root page/node and traverse to correct page.
 	c.stack = c.stack[:0]
 	c.search(seek, c.bucket.root)
-	ref := &c.stack[len(c.stack)-1]
-
-	// If the cursor is pointing to the end of page/node then return nil.
-	if ref.index >= ref.count() {
-		return nil, nil, 0
-	}
 
 	// If this is a bucket then return a nil value.
 	return c.keyValue()
@@ -339,6 +333,8 @@ func (c *Cursor) nsearch(key []byte) {
 // keyValue returns the key and value of the current leaf element.
 func (c *Cursor) keyValue() ([]byte, []byte, uint32) {
 	ref := &c.stack[len(c.stack)-1]
+
+	// If the cursor is pointing to the end of page/node then return nil.
 	if ref.count() == 0 || ref.index >= ref.count() {
 		return nil, nil, 0
 	}

backend/vendor/github.com/coreos/bbolt/db.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"errors"
@@ -7,8 +7,7 @@ import (
 	"log"
 	"os"
 	"runtime"
-	"runtime/debug"
-	"strings"
+	"sort"
 	"sync"
 	"time"
 	"unsafe"
@@ -23,6 +22,8 @@ const version = 2
 // Represents a marker value to indicate that a file is a Bolt DB.
 const magic uint32 = 0xED0CDAED
 
+const pgidNoFreelist pgid = 0xffffffffffffffff
+
 // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when
 // syncing changes to a file.  This is required as some operating systems,
 // such as OpenBSD, do not have a unified buffer cache (UBC) and writes
@@ -39,6 +40,19 @@ const (
 // default page size for db is set to the OS page size.
 var defaultPageSize = os.Getpagesize()
 
+// The time elapsed between consecutive file locking attempts.
+const flockRetryTimeout = 50 * time.Millisecond
+
+// FreelistType is the type of the freelist backend
+type FreelistType string
+
+const (
+	// FreelistArrayType indicates backend freelist type is array
+	FreelistArrayType = FreelistType("array")
+	// FreelistMapType indicates backend freelist type is hashmap
+	FreelistMapType = FreelistType("hashmap")
+)
+
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -61,6 +75,18 @@ type DB struct {
 	// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
 	NoSync bool
 
+	// When true, skips syncing freelist to disk. This improves the database
+	// write performance under normal operation, but requires a full database
+	// re-sync during recovery.
+	NoFreelistSync bool
+
+	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
+	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// The alternative one is using hashmap, it is faster in almost all circumstances
+	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
+	// The default type is array
+	FreelistType FreelistType
+
 	// When true, skips the truncate call when growing the database.
 	// Setting this to true is only safe on non-ext3/ext4 systems.
 	// Skipping truncation avoids preallocation of hard drive space and
@@ -96,8 +122,7 @@ type DB struct {
 
 	path     string
 	file     *os.File
-	lockfile *os.File // windows only
-	dataref  []byte   // mmap'ed readonly, write throws SEGV
+	dataref  []byte // mmap'ed readonly, write throws SEGV
 	data     *[maxMapSize]byte
 	datasz   int
 	filesz   int // current on disk file size
@@ -107,9 +132,11 @@ type DB struct {
 	opened   bool
 	rwtx     *Tx
 	txs      []*Tx
-	freelist *freelist
 	stats    Stats
 
+	freelist     *freelist
+	freelistLoad sync.Once
+
 	pagePool sync.Pool
 
 	batchMu sync.Mutex
@@ -148,14 +175,18 @@ func (db *DB) String() string {
 // If the file does not exist then it will be created automatically.
 // Passing in nil options will cause Bolt to open the database with the default options.
 func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
-	var db = &DB{opened: true}
-
+	db := &DB{
+		opened: true,
+	}
 	// Set default options if no options are provided.
 	if options == nil {
 		options = DefaultOptions
 	}
+	db.NoSync = options.NoSync
 	db.NoGrowSync = options.NoGrowSync
 	db.MmapFlags = options.MmapFlags
+	db.NoFreelistSync = options.NoFreelistSync
+	db.FreelistType = options.FreelistType
 
 	// Set default values for later DB operations.
 	db.MaxBatchSize = DefaultMaxBatchSize
@@ -183,7 +214,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// if !options.ReadOnly.
 	// The database file is locked using the shared lock (more than one process may
 	// hold a lock at the same time) otherwise (options.ReadOnly is set).
-	if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil {
+	if err := flock(db, !db.readOnly, options.Timeout); err != nil {
 		_ = db.close()
 		return nil, err
 	}
@@ -191,31 +222,41 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// Default values for test hooks
 	db.ops.writeAt = db.file.WriteAt
 
+	if db.pageSize = options.PageSize; db.pageSize == 0 {
+		// Set the default page size to the OS page size.
+		db.pageSize = defaultPageSize
+	}
+
 	// Initialize the database if it doesn't exist.
 	if info, err := db.file.Stat(); err != nil {
+		_ = db.close()
 		return nil, err
 	} else if info.Size() == 0 {
 		// Initialize new files with meta pages.
 		if err := db.init(); err != nil {
+			// clean up file descriptor on initialization fail
+			_ = db.close()
 			return nil, err
 		}
 	} else {
 		// Read the first meta page to determine the page size.
 		var buf [0x1000]byte
-		if _, err := db.file.ReadAt(buf[:], 0); err == nil {
-			m := db.pageInBuffer(buf[:], 0).meta()
-			if err := m.validate(); err != nil {
-				// If we can't read the page size, we can assume it's the same
-				// as the OS -- since that's how the page size was chosen in the
-				// first place.
-				//
-				// If the first page is invalid and this OS uses a different
-				// page size than what the database was created with then we
-				// are out of luck and cannot access the database.
-				db.pageSize = os.Getpagesize()
-			} else {
+		// If we can't read the page size, but can read a page, assume
+		// it's the same as the OS or one given -- since that's how the
+		// page size was chosen in the first place.
+		//
+		// If the first page is invalid and this OS uses a different
+		// page size than what the database was created with then we
+		// are out of luck and cannot access the database.
+		//
+		// TODO: scan for next page
+		if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
+			if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
 				db.pageSize = int(m.pageSize)
 			}
+		} else {
+			_ = db.close()
+			return nil, ErrInvalid
 		}
 	}
 
@@ -232,14 +273,50 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 		return nil, err
 	}
 
-	// Read in the freelist.
-	db.freelist = newFreelist()
-	db.freelist.read(db.page(db.meta().freelist))
+	if db.readOnly {
+		return db, nil
+	}
+
+	db.loadFreelist()
+
+	// Flush freelist when transitioning from no sync to sync so
+	// NoFreelistSync unaware boltdb can open the db later.
+	if !db.NoFreelistSync && !db.hasSyncedFreelist() {
+		tx, err := db.Begin(true)
+		if tx != nil {
+			err = tx.Commit()
+		}
+		if err != nil {
+			_ = db.close()
+			return nil, err
+		}
+	}
 
 	// Mark the database as opened and return.
 	return db, nil
 }
 
+// loadFreelist reads the freelist if it is synced, or reconstructs it
+// by scanning the DB if it is not synced. It assumes there are no
+// concurrent accesses being made to the freelist.
+func (db *DB) loadFreelist() {
+	db.freelistLoad.Do(func() {
+		db.freelist = newFreelist(db.FreelistType)
+		if !db.hasSyncedFreelist() {
+			// Reconstruct free list by scanning the DB.
+			db.freelist.readIDs(db.freepages())
+		} else {
+			// Read free list from freelist page.
+			db.freelist.read(db.page(db.meta().freelist))
+		}
+		db.stats.FreePageN = db.freelist.free_count()
+	})
+}
+
+func (db *DB) hasSyncedFreelist() bool {
+	return db.meta().freelist != pgidNoFreelist
+}
+
 // mmap opens the underlying memory-mapped file and initializes the meta references.
 // minsz is the minimum size that the new mmap can be.
 func (db *DB) mmap(minsz int) error {
@@ -341,9 +418,6 @@ func (db *DB) mmapSize(size int) (int, error) {
 
 // init creates a new database file and initializes its meta pages.
 func (db *DB) init() error {
-	// Set the page size to the OS page size.
-	db.pageSize = os.Getpagesize()
-
 	// Create two meta pages on a buffer.
 	buf := make([]byte, db.pageSize*4)
 	for i := 0; i < 2; i++ {
@@ -387,7 +461,8 @@ func (db *DB) init() error {
 }
 
 // Close releases all database resources.
-// All transactions must be closed before closing the database.
+// It will block waiting for any open transactions to finish
+// before closing the database and returning.
 func (db *DB) Close() error {
 	db.rwlock.Lock()
 	defer db.rwlock.Unlock()
@@ -395,8 +470,8 @@ func (db *DB) Close() error {
 	db.metalock.Lock()
 	defer db.metalock.Unlock()
 
-	db.mmaplock.RLock()
-	defer db.mmaplock.RUnlock()
+	db.mmaplock.Lock()
+	defer db.mmaplock.Unlock()
 
 	return db.close()
 }
@@ -526,21 +601,36 @@ func (db *DB) beginRWTx() (*Tx, error) {
 	t := &Tx{writable: true}
 	t.init(db)
 	db.rwtx = t
+	db.freePages()
+	return t, nil
+}
 
-	// Free any pages associated with closed read-only transactions.
-	var minid txid = 0xFFFFFFFFFFFFFFFF
-	for _, t := range db.txs {
-		if t.meta.txid < minid {
-			minid = t.meta.txid
-		}
+// freePages releases any pages associated with closed read-only transactions.
+func (db *DB) freePages() {
+	// Free all pending pages prior to earliest open transaction.
+	sort.Sort(txsById(db.txs))
+	minid := txid(0xFFFFFFFFFFFFFFFF)
+	if len(db.txs) > 0 {
+		minid = db.txs[0].meta.txid
 	}
 	if minid > 0 {
 		db.freelist.release(minid - 1)
 	}
-
-	return t, nil
+	// Release unused txid extents.
+	for _, t := range db.txs {
+		db.freelist.releaseRange(minid, t.meta.txid-1)
+		minid = t.meta.txid + 1
+	}
+	db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF))
+	// Any page both allocated and freed in an extent is safe to release.
 }
 
+type txsById []*Tx
+
+func (t txsById) Len() int           { return len(t) }
+func (t txsById) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
+func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid }
+
 // removeTx removes a transaction from the database.
 func (db *DB) removeTx(tx *Tx) {
 	// Release the read lock on the mmap.
@@ -552,7 +642,10 @@ func (db *DB) removeTx(tx *Tx) {
 	// Remove the transaction.
 	for i, t := range db.txs {
 		if t == tx {
-			db.txs = append(db.txs[:i], db.txs[i+1:]...)
+			last := len(db.txs) - 1
+			db.txs[i] = db.txs[last]
+			db.txs[last] = nil
+			db.txs = db.txs[:last]
 			break
 		}
 	}
@@ -630,11 +723,7 @@ func (db *DB) View(fn func(*Tx) error) error {
 		return err
 	}
 
-	if err := t.Rollback(); err != nil {
-		return err
-	}
-
-	return nil
+	return t.Rollback()
 }
 
 // Batch calls fn as part of a batch. It behaves similar to Update,
@@ -734,9 +823,7 @@ retry:
 
 		// pass success, or bolt internal errors, to all callers
 		for _, c := range b.calls {
-			if c.err != nil {
-				c.err <- err
-			}
+			c.err <- err
 		}
 		break retry
 	}
@@ -823,7 +910,7 @@ func (db *DB) meta() *meta {
 }
 
 // allocate returns a contiguous block of memory starting at a given page.
-func (db *DB) allocate(count int) (*page, error) {
+func (db *DB) allocate(txid txid, count int) (*page, error) {
 	// Allocate a temporary buffer for the page.
 	var buf []byte
 	if count == 1 {
@@ -835,7 +922,7 @@ func (db *DB) allocate(count int) (*page, error) {
 	p.overflow = uint32(count - 1)
 
 	// Use pages from the freelist if they are available.
-	if p.id = db.freelist.allocate(count); p.id != 0 {
+	if p.id = db.freelist.allocate(txid, count); p.id != 0 {
 		return p, nil
 	}
 
@@ -890,6 +977,38 @@ func (db *DB) IsReadOnly() bool {
 	return db.readOnly
 }
 
+func (db *DB) freepages() []pgid {
+	tx, err := db.beginTx()
+	defer func() {
+		err = tx.Rollback()
+		if err != nil {
+			panic("freepages: failed to rollback tx")
+		}
+	}()
+	if err != nil {
+		panic("freepages: failed to open read only tx")
+	}
+
+	reachable := make(map[pgid]*page)
+	nofreed := make(map[pgid]bool)
+	ech := make(chan error)
+	go func() {
+		for e := range ech {
+			panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e))
+		}
+	}()
+	tx.checkBucket(&tx.root, reachable, nofreed, ech)
+	close(ech)
+
+	var fids []pgid
+	for i := pgid(2); i < db.meta().pgid; i++ {
+		if _, ok := reachable[i]; !ok {
+			fids = append(fids, i)
+		}
+	}
+	return fids
+}
+
 // Options represents the options that can be set when opening a database.
 type Options struct {
 	// Timeout is the amount of time to wait to obtain a file lock.
@@ -900,6 +1019,17 @@ type Options struct {
 	// Sets the DB.NoGrowSync flag before memory mapping the file.
 	NoGrowSync bool
 
+	// Do not sync freelist to disk. This improves the database write performance
+	// under normal operation, but requires a full database re-sync during recovery.
+	NoFreelistSync bool
+
+	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
+	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// The alternative one is using hashmap, it is faster in almost all circumstances
+	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
+	// The default type is array
+	FreelistType FreelistType
+
 	// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
 	// grab a shared lock (UNIX).
 	ReadOnly bool
@@ -916,13 +1046,22 @@ type Options struct {
 	// If initialMmapSize is smaller than the previous database size,
 	// it takes no effect.
 	InitialMmapSize int
+
+	// PageSize overrides the default OS page size.
+	PageSize int
+
+	// NoSync sets the initial value of DB.NoSync. Normally this can just be
+	// set directly on the DB itself when returned from Open(), but this option
+	// is useful in APIs which expose Options but not the underlying DB.
+	NoSync bool
 }
 
 // DefaultOptions represent the options used if nil options are passed into Open().
 // No timeout is used which will cause Bolt to wait indefinitely for a lock.
 var DefaultOptions = &Options{
-	Timeout:    0,
-	NoGrowSync: false,
+	Timeout:      0,
+	NoGrowSync:   false,
+	FreelistType: FreelistArrayType,
 }
 
 // Stats represents statistics about the database.
@@ -952,15 +1091,11 @@ func (s *Stats) Sub(other *Stats) Stats {
 	diff.PendingPageN = s.PendingPageN
 	diff.FreeAlloc = s.FreeAlloc
 	diff.FreelistInuse = s.FreelistInuse
-	diff.TxN = other.TxN - s.TxN
+	diff.TxN = s.TxN - other.TxN
 	diff.TxStats = s.TxStats.Sub(&other.TxStats)
 	return diff
 }
 
-func (s *Stats) add(other *Stats) {
-	s.TxStats.add(&other.TxStats)
-}
-
 type Info struct {
 	Data     uintptr
 	PageSize int
@@ -999,7 +1134,8 @@ func (m *meta) copy(dest *meta) {
 func (m *meta) write(p *page) {
 	if m.root.root >= m.pgid {
 		panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid))
-	} else if m.freelist >= m.pgid {
+	} else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist {
+		// TODO: reject pgidNoFreeList if !NoFreelistSync
 		panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid))
 	}
 
@@ -1026,11 +1162,3 @@ func _assert(condition bool, msg string, v ...interface{}) {
 		panic(fmt.Sprintf("assertion failed: "+msg, v...))
 	}
 }
-
-func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
-func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
-
-func printstack() {
-	stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n")
-	fmt.Fprintln(os.Stderr, stack)
-}

backend/vendor/github.com/coreos/bbolt/doc.go

@@ -1,5 +1,5 @@
 /*
-Package bolt implements a low-level key/value store in pure Go. It supports
+package bbolt implements a low-level key/value store in pure Go. It supports
 fully serializable transactions, ACID semantics, and lock-free MVCC with
 multiple readers and a single writer. Bolt can be used for projects that
 want a simple data store without the need to add large dependencies such as
@@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic.
 
 
 */
-package bolt
+package bbolt

backend/vendor/github.com/coreos/bbolt/errors.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import "errors"
 

backend/vendor/github.com/coreos/bbolt/freelist.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -6,25 +6,72 @@ import (
 	"unsafe"
 )
 
+// txPending holds a list of pgids and corresponding allocation txns
+// that are pending to be freed.
+type txPending struct {
+	ids              []pgid
+	alloctx          []txid // txids allocating the ids
+	lastReleaseBegin txid   // beginning txid of last matching releaseRange
+}
+
+// pidSet holds the set of starting pgids which have the same span size
+type pidSet map[pgid]struct{}
+
 // freelist represents a list of all pages that are available for allocation.
 // It also tracks pages that have been freed but are still in use by open transactions.
 type freelist struct {
-	ids     []pgid          // all free and available free page ids.
-	pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
-	cache   map[pgid]bool   // fast lookup of all free and pending page ids.
+	freelistType   FreelistType                // freelist type
+	ids            []pgid                      // all free and available free page ids.
+	allocs         map[pgid]txid               // mapping of txid that allocated a pgid.
+	pending        map[txid]*txPending         // mapping of soon-to-be free page ids by tx.
+	cache          map[pgid]bool               // fast lookup of all free and pending page ids.
+	freemaps       map[uint64]pidSet           // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size
+	forwardMap     map[pgid]uint64             // key is start pgid, value is its span size
+	backwardMap    map[pgid]uint64             // key is end pgid, value is its span size
+	allocate       func(txid txid, n int) pgid // the freelist allocate func
+	free_count     func() int                  // the function which gives you free page number
+	mergeSpans     func(ids pgids)             // the mergeSpan func
+	getFreePageIDs func() []pgid               // get free pgids func
+	readIDs        func(pgids []pgid)          // readIDs func reads list of pages and init the freelist
 }
 
 // newFreelist returns an empty, initialized freelist.
-func newFreelist() *freelist {
-	return &freelist{
-		pending: make(map[txid][]pgid),
-		cache:   make(map[pgid]bool),
+func newFreelist(freelistType FreelistType) *freelist {
+	f := &freelist{
+		freelistType: freelistType,
+		allocs:       make(map[pgid]txid),
+		pending:      make(map[txid]*txPending),
+		cache:        make(map[pgid]bool),
+		freemaps:     make(map[uint64]pidSet),
+		forwardMap:   make(map[pgid]uint64),
+		backwardMap:  make(map[pgid]uint64),
 	}
+
+	if freelistType == FreelistMapType {
+		f.allocate = f.hashmapAllocate
+		f.free_count = f.hashmapFreeCount
+		f.mergeSpans = f.hashmapMergeSpans
+		f.getFreePageIDs = f.hashmapGetFreePageIDs
+		f.readIDs = f.hashmapReadIDs
+	} else {
+		f.allocate = f.arrayAllocate
+		f.free_count = f.arrayFreeCount
+		f.mergeSpans = f.arrayMergeSpans
+		f.getFreePageIDs = f.arrayGetFreePageIDs
+		f.readIDs = f.arrayReadIDs
+	}
+
+	return f
 }
 
 // size returns the size of the page after serialization.
 func (f *freelist) size() int {
-	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count())
+	n := f.count()
+	if n >= 0xFFFF {
+		// The first element will be used to store the count. See freelist.write.
+		n++
+	}
+	return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n)
 }
 
 // count returns count of pages on the freelist
@@ -32,35 +79,34 @@ func (f *freelist) count() int {
 	return f.free_count() + f.pending_count()
 }
 
-// free_count returns count of free pages
-func (f *freelist) free_count() int {
+// arrayFreeCount returns count of free pages(array version)
+func (f *freelist) arrayFreeCount() int {
 	return len(f.ids)
 }
 
 // pending_count returns count of pending pages
 func (f *freelist) pending_count() int {
 	var count int
-	for _, list := range f.pending {
-		count += len(list)
+	for _, txp := range f.pending {
+		count += len(txp.ids)
 	}
 	return count
 }
 
-// all returns a list of all free ids and all pending ids in one sorted list.
-func (f *freelist) all() []pgid {
-	m := make(pgids, 0)
-
-	for _, list := range f.pending {
-		m = append(m, list...)
+// copyall copies into dst a list of all free ids and all pending ids in one sorted list.
+// f.count returns the minimum length required for dst.
+func (f *freelist) copyall(dst []pgid) {
+	m := make(pgids, 0, f.pending_count())
+	for _, txp := range f.pending {
+		m = append(m, txp.ids...)
 	}
-
 	sort.Sort(m)
-	return pgids(f.ids).merge(m)
+	mergepgids(dst, f.getFreePageIDs(), m)
 }
 
-// allocate returns the starting page id of a contiguous list of pages of a given size.
+// arrayAllocate returns the starting page id of a contiguous list of pages of a given size.
 // If a contiguous block cannot be found then 0 is returned.
-func (f *freelist) allocate(n int) pgid {
+func (f *freelist) arrayAllocate(txid txid, n int) pgid {
 	if len(f.ids) == 0 {
 		return 0
 	}
@@ -93,7 +139,7 @@ func (f *freelist) allocate(n int) pgid {
 			for i := pgid(0); i < pgid(n); i++ {
 				delete(f.cache, initial+i)
 			}
-
+			f.allocs[initial] = txid
 			return initial
 		}
 
@@ -110,44 +156,103 @@ func (f *freelist) free(txid txid, p *page) {
 	}
 
 	// Free page and all its overflow pages.
-	var ids = f.pending[txid]
+	txp := f.pending[txid]
+	if txp == nil {
+		txp = &txPending{}
+		f.pending[txid] = txp
+	}
+	allocTxid, ok := f.allocs[p.id]
+	if ok {
+		delete(f.allocs, p.id)
+	} else if (p.flags & freelistPageFlag) != 0 {
+		// Freelist is always allocated by prior tx.
+		allocTxid = txid - 1
+	}
+
 	for id := p.id; id <= p.id+pgid(p.overflow); id++ {
 		// Verify that page is not already free.
 		if f.cache[id] {
 			panic(fmt.Sprintf("page %d already freed", id))
 		}
-
 		// Add to the freelist and cache.
-		ids = append(ids, id)
+		txp.ids = append(txp.ids, id)
+		txp.alloctx = append(txp.alloctx, allocTxid)
 		f.cache[id] = true
 	}
-	f.pending[txid] = ids
 }
 
 // release moves all page ids for a transaction id (or older) to the freelist.
 func (f *freelist) release(txid txid) {
 	m := make(pgids, 0)
-	for tid, ids := range f.pending {
+	for tid, txp := range f.pending {
 		if tid <= txid {
 			// Move transaction's pending pages to the available freelist.
 			// Don't remove from the cache since the page is still free.
-			m = append(m, ids...)
+			m = append(m, txp.ids...)
 			delete(f.pending, tid)
 		}
 	}
-	sort.Sort(m)
-	f.ids = pgids(f.ids).merge(m)
+	f.mergeSpans(m)
+}
+
+// releaseRange moves pending pages allocated within an extent [begin,end] to the free list.
+func (f *freelist) releaseRange(begin, end txid) {
+	if begin > end {
+		return
+	}
+	var m pgids
+	for tid, txp := range f.pending {
+		if tid < begin || tid > end {
+			continue
+		}
+		// Don't recompute freed pages if ranges haven't updated.
+		if txp.lastReleaseBegin == begin {
+			continue
+		}
+		for i := 0; i < len(txp.ids); i++ {
+			if atx := txp.alloctx[i]; atx < begin || atx > end {
+				continue
+			}
+			m = append(m, txp.ids[i])
+			txp.ids[i] = txp.ids[len(txp.ids)-1]
+			txp.ids = txp.ids[:len(txp.ids)-1]
+			txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1]
+			txp.alloctx = txp.alloctx[:len(txp.alloctx)-1]
+			i--
+		}
+		txp.lastReleaseBegin = begin
+		if len(txp.ids) == 0 {
+			delete(f.pending, tid)
+		}
+	}
+	f.mergeSpans(m)
 }
 
 // rollback removes the pages from a given pending tx.
 func (f *freelist) rollback(txid txid) {
 	// Remove page ids from cache.
-	for _, id := range f.pending[txid] {
-		delete(f.cache, id)
+	txp := f.pending[txid]
+	if txp == nil {
+		return
 	}
-
-	// Remove pages from pending list.
+	var m pgids
+	for i, pgid := range txp.ids {
+		delete(f.cache, pgid)
+		tx := txp.alloctx[i]
+		if tx == 0 {
+			continue
+		}
+		if tx != txid {
+			// Pending free aborted; restore page back to alloc list.
+			f.allocs[pgid] = tx
+		} else {
+			// Freed page was allocated by this txn; OK to throw away.
+			m = append(m, pgid)
+		}
+	}
+	// Remove pages from pending list and mark as free if allocated by txid.
 	delete(f.pending, txid)
+	f.mergeSpans(m)
 }
 
 // freed returns whether a given page is in the free list.
@@ -157,6 +262,9 @@ func (f *freelist) freed(pgid pgid) bool {
 
 // read initializes the freelist from a freelist page.
 func (f *freelist) read(p *page) {
+	if (p.flags & freelistPageFlag) == 0 {
+		panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ()))
+	}
 	// If the page.count is at the max uint16 value (64k) then it's considered
 	// an overflow and the size of the freelist is stored as the first element.
 	idx, count := 0, int(p.count)
@@ -169,39 +277,49 @@ func (f *freelist) read(p *page) {
 	if count == 0 {
 		f.ids = nil
 	} else {
-		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count]
-		f.ids = make([]pgid, len(ids))
-		copy(f.ids, ids)
+		ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count]
 
+		// copy the ids, so we don't modify on the freelist page directly
+		idsCopy := make([]pgid, count)
+		copy(idsCopy, ids)
 		// Make sure they're sorted.
-		sort.Sort(pgids(f.ids))
-	}
+		sort.Sort(pgids(idsCopy))
 
-	// Rebuild the page cache.
+		f.readIDs(idsCopy)
+	}
+}
+
+// arrayReadIDs initializes the freelist from a given list of ids.
+func (f *freelist) arrayReadIDs(ids []pgid) {
+	f.ids = ids
 	f.reindex()
 }
 
+func (f *freelist) arrayGetFreePageIDs() []pgid {
+	return f.ids
+}
+
 // write writes the page ids onto a freelist page. All free and pending ids are
 // saved to disk since in the event of a program crash, all pending ids will
 // become free.
 func (f *freelist) write(p *page) error {
 	// Combine the old free pgids and pgids waiting on an open transaction.
-	ids := f.all()
 
 	// Update the header flag.
 	p.flags |= freelistPageFlag
 
 	// The page.count can only hold up to 64k elements so if we overflow that
 	// number then we handle it by putting the size in the first element.
-	if len(ids) == 0 {
-		p.count = uint16(len(ids))
-	} else if len(ids) < 0xFFFF {
-		p.count = uint16(len(ids))
-		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids)
+	lenids := f.count()
+	if lenids == 0 {
+		p.count = uint16(lenids)
+	} else if lenids < 0xFFFF {
+		p.count = uint16(lenids)
+		f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:])
 	} else {
 		p.count = 0xFFFF
-		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids))
-		copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids)
+		((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids)
+		f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:])
 	}
 
 	return nil
@@ -213,8 +331,8 @@ func (f *freelist) reload(p *page) {
 
 	// Build a cache of only pending pages.
 	pcache := make(map[pgid]bool)
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			pcache[pendingID] = true
 		}
 	}
@@ -222,27 +340,31 @@ func (f *freelist) reload(p *page) {
 	// Check each page in the freelist and build a new available freelist
 	// with any pages not in the pending lists.
 	var a []pgid
-	for _, id := range f.ids {
+	for _, id := range f.getFreePageIDs() {
 		if !pcache[id] {
 			a = append(a, id)
 		}
 	}
-	f.ids = a
 
-	// Once the available list is rebuilt then rebuild the free cache so that
-	// it includes the available and pending free pages.
-	f.reindex()
+	f.readIDs(a)
 }
 
 // reindex rebuilds the free cache based on available and pending free lists.
 func (f *freelist) reindex() {
-	f.cache = make(map[pgid]bool)
-	for _, id := range f.ids {
+	ids := f.getFreePageIDs()
+	f.cache = make(map[pgid]bool, len(ids))
+	for _, id := range ids {
 		f.cache[id] = true
 	}
-	for _, pendingIDs := range f.pending {
-		for _, pendingID := range pendingIDs {
+	for _, txp := range f.pending {
+		for _, pendingID := range txp.ids {
 			f.cache[pendingID] = true
 		}
 	}
 }
+
+// arrayMergeSpans try to merge list of pages(represented by pgids) with existing spans but using array
+func (f *freelist) arrayMergeSpans(ids pgids) {
+	sort.Sort(ids)
+	f.ids = pgids(f.ids).merge(ids)
+}

backend/vendor/github.com/coreos/bbolt/freelist_hmap.go

@@ -0,0 +1,178 @@
+package bbolt
+
+import "sort"
+
+// hashmapFreeCount returns count of free pages(hashmap version)
+func (f *freelist) hashmapFreeCount() int {
+	// use the forwardmap to get the total count
+	count := 0
+	for _, size := range f.forwardMap {
+		count += int(size)
+	}
+	return count
+}
+
+// hashmapAllocate serves the same purpose as arrayAllocate, but use hashmap as backend
+func (f *freelist) hashmapAllocate(txid txid, n int) pgid {
+	if n == 0 {
+		return 0
+	}
+
+	// if we have a exact size match just return short path
+	if bm, ok := f.freemaps[uint64(n)]; ok {
+		for pid := range bm {
+			// remove the span
+			f.delSpan(pid, uint64(n))
+
+			f.allocs[pid] = txid
+
+			for i := pgid(0); i < pgid(n); i++ {
+				delete(f.cache, pid+pgid(i))
+			}
+			return pid
+		}
+	}
+
+	// lookup the map to find larger span
+	for size, bm := range f.freemaps {
+		if size < uint64(n) {
+			continue
+		}
+
+		for pid := range bm {
+			// remove the initial
+			f.delSpan(pid, uint64(size))
+
+			f.allocs[pid] = txid
+
+			remain := size - uint64(n)
+
+			// add remain span
+			f.addSpan(pid+pgid(n), remain)
+
+			for i := pgid(0); i < pgid(n); i++ {
+				delete(f.cache, pid+pgid(i))
+			}
+			return pid
+		}
+	}
+
+	return 0
+}
+
+// hashmapReadIDs reads pgids as input an initial the freelist(hashmap version)
+func (f *freelist) hashmapReadIDs(pgids []pgid) {
+	f.init(pgids)
+
+	// Rebuild the page cache.
+	f.reindex()
+}
+
+// hashmapGetFreePageIDs returns the sorted free page ids
+func (f *freelist) hashmapGetFreePageIDs() []pgid {
+	count := f.free_count()
+	if count == 0 {
+		return nil
+	}
+
+	m := make([]pgid, 0, count)
+	for start, size := range f.forwardMap {
+		for i := 0; i < int(size); i++ {
+			m = append(m, start+pgid(i))
+		}
+	}
+	sort.Sort(pgids(m))
+
+	return m
+}
+
+// hashmapMergeSpans try to merge list of pages(represented by pgids) with existing spans
+func (f *freelist) hashmapMergeSpans(ids pgids) {
+	for _, id := range ids {
+		// try to see if we can merge and update
+		f.mergeWithExistingSpan(id)
+	}
+}
+
+// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward
+func (f *freelist) mergeWithExistingSpan(pid pgid) {
+	prev := pid - 1
+	next := pid + 1
+
+	preSize, mergeWithPrev := f.backwardMap[prev]
+	nextSize, mergeWithNext := f.forwardMap[next]
+	newStart := pid
+	newSize := uint64(1)
+
+	if mergeWithPrev {
+		//merge with previous span
+		start := prev + 1 - pgid(preSize)
+		f.delSpan(start, preSize)
+
+		newStart -= pgid(preSize)
+		newSize += preSize
+	}
+
+	if mergeWithNext {
+		// merge with next span
+		f.delSpan(next, nextSize)
+		newSize += nextSize
+	}
+
+	f.addSpan(newStart, newSize)
+}
+
+func (f *freelist) addSpan(start pgid, size uint64) {
+	f.backwardMap[start-1+pgid(size)] = size
+	f.forwardMap[start] = size
+	if _, ok := f.freemaps[size]; !ok {
+		f.freemaps[size] = make(map[pgid]struct{})
+	}
+
+	f.freemaps[size][start] = struct{}{}
+}
+
+func (f *freelist) delSpan(start pgid, size uint64) {
+	delete(f.forwardMap, start)
+	delete(f.backwardMap, start+pgid(size-1))
+	delete(f.freemaps[size], start)
+	if len(f.freemaps[size]) == 0 {
+		delete(f.freemaps, size)
+	}
+}
+
+// initial from pgids using when use hashmap version
+// pgids must be sorted
+func (f *freelist) init(pgids []pgid) {
+	if len(pgids) == 0 {
+		return
+	}
+
+	size := uint64(1)
+	start := pgids[0]
+
+	if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) {
+		panic("pgids not sorted")
+	}
+
+	f.freemaps = make(map[uint64]pidSet)
+	f.forwardMap = make(map[pgid]uint64)
+	f.backwardMap = make(map[pgid]uint64)
+
+	for i := 1; i < len(pgids); i++ {
+		// continuous page
+		if pgids[i] == pgids[i-1]+1 {
+			size++
+		} else {
+			f.addSpan(start, size)
+
+			size = 1
+			start = pgids[i]
+		}
+	}
+
+	// init the tail
+	if size != 0 && start != 0 {
+		f.addSpan(start, size)
+	}
+}

backend/vendor/github.com/coreos/bbolt/node.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"bytes"
@@ -365,7 +365,7 @@ func (n *node) spill() error {
 		}
 
 		// Allocate contiguous space for the node.
-		p, err := tx.allocate((node.size() / tx.db.pageSize) + 1)
+		p, err := tx.allocate((node.size() + tx.db.pageSize - 1) / tx.db.pageSize)
 		if err != nil {
 			return err
 		}

backend/vendor/github.com/coreos/bbolt/page.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -145,12 +145,33 @@ func (a pgids) merge(b pgids) pgids {
 	// Return the opposite slice if one is nil.
 	if len(a) == 0 {
 		return b
-	} else if len(b) == 0 {
+	}
+	if len(b) == 0 {
 		return a
 	}
+	merged := make(pgids, len(a)+len(b))
+	mergepgids(merged, a, b)
+	return merged
+}
 
-	// Create a list to hold all elements from both lists.
-	merged := make(pgids, 0, len(a)+len(b))
+// mergepgids copies the sorted union of a and b into dst.
+// If dst is too small, it panics.
+func mergepgids(dst, a, b pgids) {
+	if len(dst) < len(a)+len(b) {
+		panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b)))
+	}
+	// Copy in the opposite slice if one is nil.
+	if len(a) == 0 {
+		copy(dst, b)
+		return
+	}
+	if len(b) == 0 {
+		copy(dst, a)
+		return
+	}
+
+	// Merged will hold all elements from both lists.
+	merged := dst[:0]
 
 	// Assign lead to the slice with a lower starting value, follow to the higher value.
 	lead, follow := a, b
@@ -172,7 +193,5 @@ func (a pgids) merge(b pgids) pgids {
 	}
 
 	// Append what's left in follow.
-	merged = append(merged, follow...)
-
-	return merged
+	_ = append(merged, follow...)
 }

backend/vendor/github.com/coreos/bbolt/tx.go

@@ -1,4 +1,4 @@
-package bolt
+package bbolt
 
 import (
 	"fmt"
@@ -126,10 +126,7 @@ func (tx *Tx) DeleteBucket(name []byte) error {
 // the error is returned to the caller.
 func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error {
 	return tx.root.ForEach(func(k, v []byte) error {
-		if err := fn(k, tx.root.Bucket(k)); err != nil {
-			return err
-		}
-		return nil
+		return fn(k, tx.root.Bucket(k))
 	})
 }
 
@@ -169,28 +166,18 @@ func (tx *Tx) Commit() error {
 	// Free the old root bucket.
 	tx.meta.root.root = tx.root.root
 
-	opgid := tx.meta.pgid
-
-	// Free the freelist and allocate new pages for it. This will overestimate
-	// the size of the freelist but not underestimate the size (which would be bad).
-	tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
-	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
-	if err != nil {
-		tx.rollback()
-		return err
+	// Free the old freelist because commit writes out a fresh freelist.
+	if tx.meta.freelist != pgidNoFreelist {
+		tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
 	}
-	if err := tx.db.freelist.write(p); err != nil {
-		tx.rollback()
-		return err
-	}
-	tx.meta.freelist = p.id
 
-	// If the high water mark has moved up then attempt to grow the database.
-	if tx.meta.pgid > opgid {
-		if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
-			tx.rollback()
+	if !tx.db.NoFreelistSync {
+		err := tx.commitFreelist()
+		if err != nil {
 			return err
 		}
+	} else {
+		tx.meta.freelist = pgidNoFreelist
 	}
 
 	// Write dirty pages to disk.
@@ -235,6 +222,31 @@ func (tx *Tx) Commit() error {
 	return nil
 }
 
+func (tx *Tx) commitFreelist() error {
+	// Allocate new pages for the new free list. This will overestimate
+	// the size of the freelist but not underestimate the size (which would be bad).
+	opgid := tx.meta.pgid
+	p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
+	if err != nil {
+		tx.rollback()
+		return err
+	}
+	if err := tx.db.freelist.write(p); err != nil {
+		tx.rollback()
+		return err
+	}
+	tx.meta.freelist = p.id
+	// If the high water mark has moved up then attempt to grow the database.
+	if tx.meta.pgid > opgid {
+		if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
+			tx.rollback()
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Rollback closes the transaction and ignores all previous updates. Read-only
 // transactions must be rolled back and not committed.
 func (tx *Tx) Rollback() error {
@@ -291,7 +303,9 @@ func (tx *Tx) close() {
 }
 
 // Copy writes the entire database to a writer.
-// This function exists for backwards compatibility. Use WriteTo() instead.
+// This function exists for backwards compatibility.
+//
+// Deprecated; Use WriteTo() instead.
 func (tx *Tx) Copy(w io.Writer) error {
 	_, err := tx.WriteTo(w)
 	return err
@@ -305,7 +319,11 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 	if err != nil {
 		return 0, err
 	}
-	defer func() { _ = f.Close() }()
+	defer func() {
+		if cerr := f.Close(); err == nil {
+			err = cerr
+		}
+	}()
 
 	// Generate a meta page. We use the same page data for both meta pages.
 	buf := make([]byte, tx.db.pageSize)
@@ -333,7 +351,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 	}
 
 	// Move past the meta pages in the file.
-	if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil {
+	if _, err := f.Seek(int64(tx.db.pageSize*2), io.SeekStart); err != nil {
 		return n, fmt.Errorf("seek: %s", err)
 	}
 
@@ -344,7 +362,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
 		return n, err
 	}
 
-	return n, f.Close()
+	return n, nil
 }
 
 // CopyFile copies the entire database to file at the given path.
@@ -379,9 +397,14 @@ func (tx *Tx) Check() <-chan error {
 }
 
 func (tx *Tx) check(ch chan error) {
+	// Force loading free list if opened in ReadOnly mode.
+	tx.db.loadFreelist()
+
 	// Check if any pages are double freed.
 	freed := make(map[pgid]bool)
-	for _, id := range tx.db.freelist.all() {
+	all := make([]pgid, tx.db.freelist.count())
+	tx.db.freelist.copyall(all)
+	for _, id := range all {
 		if freed[id] {
 			ch <- fmt.Errorf("page %d: already freed", id)
 		}
@@ -392,8 +415,10 @@ func (tx *Tx) check(ch chan error) {
 	reachable := make(map[pgid]*page)
 	reachable[0] = tx.page(0) // meta0
 	reachable[1] = tx.page(1) // meta1
-	for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
-		reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
+	if tx.meta.freelist != pgidNoFreelist {
+		for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
+			reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
+		}
 	}
 
 	// Recursively check buckets.
@@ -451,7 +476,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo
 
 // allocate returns a contiguous block of memory starting at a given page.
 func (tx *Tx) allocate(count int) (*page, error) {
-	p, err := tx.db.allocate(count)
+	p, err := tx.db.allocate(tx.meta.txid, count)
 	if err != nil {
 		return nil, err
 	}
@@ -460,7 +485,7 @@ func (tx *Tx) allocate(count int) (*page, error) {
 	tx.pages[p.id] = p
 
 	// Update statistics.
-	tx.stats.PageCount++
+	tx.stats.PageCount += count
 	tx.stats.PageAlloc += count * tx.db.pageSize
 
 	return p, nil

backend/vendor/github.com/go-pkgz/auth/auth.go

@@ -44,6 +44,7 @@ type Opts struct {
 	JWTHeaderKey   string // default "X-JWT"
 	XSRFCookieName string // default "XSRF-TOKEN"
 	XSRFHeaderKey  string // default "X-XSRF-TOKEN"
+	JWTQuery       string // default "token"
 
 	Issuer string // optional value for iss claim, usually the application name, default "go-pkgz/auth"
 
@@ -94,6 +95,7 @@ func NewService(opts Opts) (res *Service) {
 		JWTHeaderKey:   opts.JWTHeaderKey,
 		XSRFCookieName: opts.XSRFCookieName,
 		XSRFHeaderKey:  opts.XSRFHeaderKey,
+		JWTQuery:       opts.JWTQuery,
 		Issuer:         res.issuer,
 		AudienceReader: opts.AudienceReader,
 	})

backend/vendor/github.com/go-pkgz/auth/avatar/store.go

@@ -4,14 +4,13 @@ package avatar
 
 import (
 	"crypto/sha1"
-	"strings"
-
-	// Initializing packages for supporting GIF and JPEG formats.
-	_ "image/gif"
-	_ "image/jpeg"
+	_ "image/gif"  // initializing packages for supporting GIF
+	_ "image/jpeg" // initializing packages for supporting JPEG.
+	_ "image/png"  // initializing packages for supporting PNG.
 	"io"
 	"log"
 	"regexp"
+	"strings"
 
 	"github.com/go-pkgz/auth/token"
 )

backend/vendor/github.com/go-pkgz/auth/provider/dev_provider.go

@@ -9,10 +9,10 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/auth/logger"
 	"golang.org/x/oauth2"
 
+	"github.com/go-pkgz/auth/avatar"
+	"github.com/go-pkgz/auth/logger"
 	"github.com/go-pkgz/auth/token"
 )
 

backend/vendor/github.com/go-pkgz/auth/provider/direct.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"net/http"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/go-pkgz/rest"
 
 	"github.com/go-pkgz/auth/logger"

backend/vendor/github.com/go-pkgz/auth/token/jwt.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 	"github.com/pkg/errors"
 )
 
@@ -32,16 +32,19 @@ type Handshake struct {
 	ID    string `json:"id,omitempty"`
 }
 
-// default names for cookies and headers
 const (
-	jwtCookieName  = "JWT"
-	jwtHeaderKey   = "X-JWT"
-	xsrfCookieName = "XSRF-TOKEN"
-	xsrfHeaderKey  = "X-XSRF-TOKEN"
-	tokenQuery     = "token"
-	issuer         = "go-pkgz/auth"
-	tokenDuration  = time.Minute * 15
-	cookieDuration = time.Hour * 24 * 31
+	// default names for cookies and headers
+	defaultJWTCookieName  = "JWT"
+	defaultJWTHeaderKey   = "X-JWT"
+	defaultXSRFCookieName = "XSRF-TOKEN"
+	defaultXSRFHeaderKey  = "X-XSRF-TOKEN"
+
+	defaultIssuer = "go-pkgz/auth"
+
+	defaultTokenDuration  = time.Minute * 15
+	defaultCookieDuration = time.Hour * 24 * 31
+
+	defaultTokenQuery = "token"
 )
 
 // Opts holds constructor params
@@ -58,7 +61,7 @@ type Opts struct {
 	JWTHeaderKey   string
 	XSRFCookieName string
 	XSRFHeaderKey  string
-
+	JWTQuery       string
 	AudienceReader Audience // allowed aud values
 	Issuer         string   // optional value for iss claim, usually application name
 }
@@ -73,18 +76,19 @@ func NewService(opts Opts) *Service {
 		}
 	}
 
-	setDefault(&res.JWTCookieName, jwtCookieName)
-	setDefault(&res.JWTHeaderKey, jwtHeaderKey)
-	setDefault(&res.XSRFCookieName, xsrfCookieName)
-	setDefault(&res.XSRFHeaderKey, xsrfHeaderKey)
-	setDefault(&res.Issuer, issuer)
+	setDefault(&res.JWTCookieName, defaultJWTCookieName)
+	setDefault(&res.JWTHeaderKey, defaultJWTHeaderKey)
+	setDefault(&res.XSRFCookieName, defaultXSRFCookieName)
+	setDefault(&res.XSRFHeaderKey, defaultXSRFHeaderKey)
+	setDefault(&res.JWTQuery, defaultTokenQuery)
+	setDefault(&res.Issuer, defaultIssuer)
 
 	if opts.TokenDuration == 0 {
-		res.TokenDuration = tokenDuration
+		res.TokenDuration = defaultTokenDuration
 	}
 
 	if opts.CookieDuration == 0 {
-		res.CookieDuration = cookieDuration
+		res.CookieDuration = defaultCookieDuration
 	}
 
 	return &res
@@ -164,12 +168,12 @@ func (j *Service) validate(claims *Claims) error {
 	}
 
 	if e, ok := cerr.(*jwt.ValidationError); ok {
-		e.Errors ^= jwt.ValidationErrorExpired // clear ValidationErrorExpired, allow expired token
-		if e.Errors != 0 {
-			return e
+		if e.Errors == jwt.ValidationErrorExpired {
+			return nil // allow expired tokens
 		}
 	}
-	return nil
+
+	return cerr
 }
 
 // Set creates token cookie with xsrf cookie and put it to ResponseWriter
@@ -198,11 +202,11 @@ func (j *Service) Set(w http.ResponseWriter, claims Claims) (Claims, error) {
 		cookieExpiration = int(j.CookieDuration.Seconds())
 	}
 
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: tokenString, HttpOnly: true, Path: "/",
+	jwtCookie := http.Cookie{Name: j.JWTCookieName, Value: tokenString, HttpOnly: true, Path: "/",
 		MaxAge: cookieExpiration, Secure: j.SecureCookies}
 	http.SetCookie(w, &jwtCookie)
 
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: claims.Id, HttpOnly: false, Path: "/",
+	xsrfCookie := http.Cookie{Name: j.XSRFCookieName, Value: claims.Id, HttpOnly: false, Path: "/",
 		MaxAge: cookieExpiration, Secure: j.SecureCookies}
 	http.SetCookie(w, &xsrfCookie)
 
@@ -217,19 +221,19 @@ func (j *Service) Get(r *http.Request) (Claims, string, error) {
 	tokenString := ""
 
 	// try to get from "token" query param
-	if tkQuery := r.URL.Query().Get(tokenQuery); tkQuery != "" {
+	if tkQuery := r.URL.Query().Get(j.JWTQuery); tkQuery != "" {
 		tokenString = tkQuery
 	}
 
-	// try to get from X-JWT header
-	if tokenHeader := r.Header.Get(jwtHeaderKey); tokenHeader != "" && tokenString == "" {
+	// try to get from JWT header
+	if tokenHeader := r.Header.Get(j.JWTHeaderKey); tokenHeader != "" && tokenString == "" {
 		tokenString = tokenHeader
 	}
 
 	// try to get from JWT cookie
 	if tokenString == "" {
 		fromCookie = true
-		jc, err := r.Cookie(jwtCookieName)
+		jc, err := r.Cookie(j.JWTCookieName)
 		if err != nil {
 			return Claims{}, "", errors.Wrap(err, "token cookie was not presented")
 		}
@@ -250,7 +254,7 @@ func (j *Service) Get(r *http.Request) (Claims, string, error) {
 	}
 
 	if fromCookie && claims.User != nil {
-		xsrf := r.Header.Get(xsrfHeaderKey)
+		xsrf := r.Header.Get(j.XSRFHeaderKey)
 		if claims.Id != xsrf {
 			return Claims{}, "", errors.New("xsrf mismatch")
 		}
@@ -265,11 +269,11 @@ func (j *Service) IsExpired(claims Claims) bool {
 
 // Reset token's cookies
 func (j *Service) Reset(w http.ResponseWriter) {
-	jwtCookie := http.Cookie{Name: jwtCookieName, Value: "", HttpOnly: false, Path: "/",
+	jwtCookie := http.Cookie{Name: j.JWTCookieName, Value: "", HttpOnly: false, Path: "/",
 		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.SecureCookies}
 	http.SetCookie(w, &jwtCookie)
 
-	xsrfCookie := http.Cookie{Name: xsrfCookieName, Value: "", HttpOnly: false, Path: "/",
+	xsrfCookie := http.Cookie{Name: j.XSRFCookieName, Value: "", HttpOnly: false, Path: "/",
 		MaxAge: -1, Expires: time.Unix(0, 0), Secure: j.SecureCookies}
 	http.SetCookie(w, &xsrfCookie)
 }

backend/vendor/github.com/go-pkgz/rest/README.md

@@ -38,8 +38,22 @@ pong
 
 ### Logger middleware
 
-Logs all info about request, including user, method, status code, response size, url, elapsed time, request body (optional).
-Can be customized by passing flags - LogNone, LogAll, LogUser and LogBody. Flags can be combined (provided multiple times)
+Logs request, request handling time and response. Log record fields in order of occurrence:
+
+- Request's HTTP method
+- Requested URL (with sanitized query)
+- Remote IP
+- Response's HTTP status code
+- Response body size
+- Request handling time
+- Userinfo associated with the request (optional)
+- Request subject (optional)
+- Request ID (if `X-Request-ID` present)
+- Request body (optional)
+
+_remote IP can be masked with user defined function_
+
+example: `019/03/05 17:26:12.976 [INFO] GET - /api/v1/find?site=remark - 8e228e9cfece - 200 (115) - 4.47784618s`
 
 ### Recoverer middleware
 

backend/vendor/github.com/golang/mock/AUTHORS

@@ -1,12 +0,0 @@
-# This is the official list of GoMock authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as
-#	Name or Organization <email address>
-# The email address is not required for organizations.
-
-# Please keep the list sorted.
-
-Alex Reece <awreece@gmail.com>
-Google Inc.

backend/vendor/github.com/golang/mock/CONTRIBUTORS

@@ -1,37 +0,0 @@
-# This is the official list of people who can contribute (and typically
-# have contributed) code to the gomock repository.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# The submission process automatically checks to make sure
-# that people submitting code are listed in this file (by email address).
-#
-# Names should be added to this file only after verifying that
-# the individual or the individual's organization has agreed to
-# the appropriate Contributor License Agreement, found here:
-#
-#     http://code.google.com/legal/individual-cla-v1.0.html
-#     http://code.google.com/legal/corporate-cla-v1.0.html
-#
-# The agreement for individuals can be filled out on the web.
-#
-# When adding J Random Contributor's name to this file,
-# either J's name or J's organization's name should be
-# added to the AUTHORS file, depending on whether the
-# individual or corporate CLA was used.
-
-# Names should be added to this file like so:
-#     Name <email address>
-#
-# An entry with two email addresses specifies that the
-# first address should be used in the submit logs and
-# that the second address should be recognized as the
-# same person when interacting with Rietveld.
-
-# Please keep the list sorted.
-
-Aaron Jacobs <jacobsa@google.com> <aaronjjacobs@gmail.com>
-Alex Reece <awreece@gmail.com>
-David Symonds <dsymonds@golang.org>
-Ryan Barrett <ryanb@google.com>

backend/vendor/github.com/golang/mock/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

backend/vendor/github.com/golang/mock/gomock/call.go

@@ -1,420 +0,0 @@
-// Copyright 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gomock
-
-import (
-	"fmt"
-	"reflect"
-	"strconv"
-	"strings"
-)
-
-// Call represents an expected call to a mock.
-type Call struct {
-	t TestHelper // for triggering test failures on invalid call setup
-
-	receiver   interface{}  // the receiver of the method call
-	method     string       // the name of the method
-	methodType reflect.Type // the type of the method
-	args       []Matcher    // the args
-	origin     string       // file and line number of call setup
-
-	preReqs []*Call // prerequisite calls
-
-	// Expectations
-	minCalls, maxCalls int
-
-	numCalls int // actual number made
-
-	// actions are called when this Call is called. Each action gets the args and
-	// can set the return values by returning a non-nil slice. Actions run in the
-	// order they are created.
-	actions []func([]interface{}) []interface{}
-}
-
-// newCall creates a *Call. It requires the method type in order to support
-// unexported methods.
-func newCall(t TestHelper, receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
-	t.Helper()
-
-	// TODO: check arity, types.
-	margs := make([]Matcher, len(args))
-	for i, arg := range args {
-		if m, ok := arg.(Matcher); ok {
-			margs[i] = m
-		} else if arg == nil {
-			// Handle nil specially so that passing a nil interface value
-			// will match the typed nils of concrete args.
-			margs[i] = Nil()
-		} else {
-			margs[i] = Eq(arg)
-		}
-	}
-
-	origin := callerInfo(3)
-	actions := []func([]interface{}) []interface{}{func([]interface{}) []interface{} {
-		// Synthesize the zero value for each of the return args' types.
-		rets := make([]interface{}, methodType.NumOut())
-		for i := 0; i < methodType.NumOut(); i++ {
-			rets[i] = reflect.Zero(methodType.Out(i)).Interface()
-		}
-		return rets
-	}}
-	return &Call{t: t, receiver: receiver, method: method, methodType: methodType,
-		args: margs, origin: origin, minCalls: 1, maxCalls: 1, actions: actions}
-}
-
-// AnyTimes allows the expectation to be called 0 or more times
-func (c *Call) AnyTimes() *Call {
-	c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity
-	return c
-}
-
-// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also
-// sets the maximum number of calls to infinity.
-func (c *Call) MinTimes(n int) *Call {
-	c.minCalls = n
-	if c.maxCalls == 1 {
-		c.maxCalls = 1e8
-	}
-	return c
-}
-
-// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also
-// sets the minimum number of calls to 0.
-func (c *Call) MaxTimes(n int) *Call {
-	c.maxCalls = n
-	if c.minCalls == 1 {
-		c.minCalls = 0
-	}
-	return c
-}
-
-// DoAndReturn declares the action to run when the call is matched.
-// The return values from this function are returned by the mocked function.
-// It takes an interface{} argument to support n-arity functions.
-func (c *Call) DoAndReturn(f interface{}) *Call {
-	// TODO: Check arity and types here, rather than dying badly elsewhere.
-	v := reflect.ValueOf(f)
-
-	c.addAction(func(args []interface{}) []interface{} {
-		vargs := make([]reflect.Value, len(args))
-		ft := v.Type()
-		for i := 0; i < len(args); i++ {
-			if args[i] != nil {
-				vargs[i] = reflect.ValueOf(args[i])
-			} else {
-				// Use the zero value for the arg.
-				vargs[i] = reflect.Zero(ft.In(i))
-			}
-		}
-		vrets := v.Call(vargs)
-		rets := make([]interface{}, len(vrets))
-		for i, ret := range vrets {
-			rets[i] = ret.Interface()
-		}
-		return rets
-	})
-	return c
-}
-
-// Do declares the action to run when the call is matched. The function's
-// return values are ignored to retain backward compatibility. To use the
-// return values call DoAndReturn.
-// It takes an interface{} argument to support n-arity functions.
-func (c *Call) Do(f interface{}) *Call {
-	// TODO: Check arity and types here, rather than dying badly elsewhere.
-	v := reflect.ValueOf(f)
-
-	c.addAction(func(args []interface{}) []interface{} {
-		vargs := make([]reflect.Value, len(args))
-		ft := v.Type()
-		for i := 0; i < len(args); i++ {
-			if args[i] != nil {
-				vargs[i] = reflect.ValueOf(args[i])
-			} else {
-				// Use the zero value for the arg.
-				vargs[i] = reflect.Zero(ft.In(i))
-			}
-		}
-		v.Call(vargs)
-		return nil
-	})
-	return c
-}
-
-// Return declares the values to be returned by the mocked function call.
-func (c *Call) Return(rets ...interface{}) *Call {
-	c.t.Helper()
-
-	mt := c.methodType
-	if len(rets) != mt.NumOut() {
-		c.t.Fatalf("wrong number of arguments to Return for %T.%v: got %d, want %d [%s]",
-			c.receiver, c.method, len(rets), mt.NumOut(), c.origin)
-	}
-	for i, ret := range rets {
-		if got, want := reflect.TypeOf(ret), mt.Out(i); got == want {
-			// Identical types; nothing to do.
-		} else if got == nil {
-			// Nil needs special handling.
-			switch want.Kind() {
-			case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
-				// ok
-			default:
-				c.t.Fatalf("argument %d to Return for %T.%v is nil, but %v is not nillable [%s]",
-					i, c.receiver, c.method, want, c.origin)
-			}
-		} else if got.AssignableTo(want) {
-			// Assignable type relation. Make the assignment now so that the generated code
-			// can return the values with a type assertion.
-			v := reflect.New(want).Elem()
-			v.Set(reflect.ValueOf(ret))
-			rets[i] = v.Interface()
-		} else {
-			c.t.Fatalf("wrong type of argument %d to Return for %T.%v: %v is not assignable to %v [%s]",
-				i, c.receiver, c.method, got, want, c.origin)
-		}
-	}
-
-	c.addAction(func([]interface{}) []interface{} {
-		return rets
-	})
-
-	return c
-}
-
-// Times declares the exact number of times a function call is expected to be executed.
-func (c *Call) Times(n int) *Call {
-	c.minCalls, c.maxCalls = n, n
-	return c
-}
-
-// SetArg declares an action that will set the nth argument's value,
-// indirected through a pointer. Or, in the case of a slice, SetArg
-// will copy value's elements into the nth argument.
-func (c *Call) SetArg(n int, value interface{}) *Call {
-	c.t.Helper()
-
-	mt := c.methodType
-	// TODO: This will break on variadic methods.
-	// We will need to check those at invocation time.
-	if n < 0 || n >= mt.NumIn() {
-		c.t.Fatalf("SetArg(%d, ...) called for a method with %d args [%s]",
-			n, mt.NumIn(), c.origin)
-	}
-	// Permit setting argument through an interface.
-	// In the interface case, we don't (nay, can't) check the type here.
-	at := mt.In(n)
-	switch at.Kind() {
-	case reflect.Ptr:
-		dt := at.Elem()
-		if vt := reflect.TypeOf(value); !vt.AssignableTo(dt) {
-			c.t.Fatalf("SetArg(%d, ...) argument is a %v, not assignable to %v [%s]",
-				n, vt, dt, c.origin)
-		}
-	case reflect.Interface:
-		// nothing to do
-	case reflect.Slice:
-		// nothing to do
-	default:
-		c.t.Fatalf("SetArg(%d, ...) referring to argument of non-pointer non-interface non-slice type %v [%s]",
-			n, at, c.origin)
-	}
-
-	c.addAction(func(args []interface{}) []interface{} {
-		v := reflect.ValueOf(value)
-		switch reflect.TypeOf(args[n]).Kind() {
-		case reflect.Slice:
-			setSlice(args[n], v)
-		default:
-			reflect.ValueOf(args[n]).Elem().Set(v)
-		}
-		return nil
-	})
-	return c
-}
-
-// isPreReq returns true if other is a direct or indirect prerequisite to c.
-func (c *Call) isPreReq(other *Call) bool {
-	for _, preReq := range c.preReqs {
-		if other == preReq || preReq.isPreReq(other) {
-			return true
-		}
-	}
-	return false
-}
-
-// After declares that the call may only match after preReq has been exhausted.
-func (c *Call) After(preReq *Call) *Call {
-	c.t.Helper()
-
-	if c == preReq {
-		c.t.Fatalf("A call isn't allowed to be its own prerequisite")
-	}
-	if preReq.isPreReq(c) {
-		c.t.Fatalf("Loop in call order: %v is a prerequisite to %v (possibly indirectly).", c, preReq)
-	}
-
-	c.preReqs = append(c.preReqs, preReq)
-	return c
-}
-
-// Returns true if the minimum number of calls have been made.
-func (c *Call) satisfied() bool {
-	return c.numCalls >= c.minCalls
-}
-
-// Returns true iff the maximum number of calls have been made.
-func (c *Call) exhausted() bool {
-	return c.numCalls >= c.maxCalls
-}
-
-func (c *Call) String() string {
-	args := make([]string, len(c.args))
-	for i, arg := range c.args {
-		args[i] = arg.String()
-	}
-	arguments := strings.Join(args, ", ")
-	return fmt.Sprintf("%T.%v(%s) %s", c.receiver, c.method, arguments, c.origin)
-}
-
-// Tests if the given call matches the expected call.
-// If yes, returns nil. If no, returns error with message explaining why it does not match.
-func (c *Call) matches(args []interface{}) error {
-	if !c.methodType.IsVariadic() {
-		if len(args) != len(c.args) {
-			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
-				c.origin, len(args), len(c.args))
-		}
-
-		for i, m := range c.args {
-			if !m.Matches(args[i]) {
-				return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
-					c.origin, strconv.Itoa(i), args[i], m)
-			}
-		}
-	} else {
-		if len(c.args) < c.methodType.NumIn()-1 {
-			return fmt.Errorf("Expected call at %s has the wrong number of matchers. Got: %d, want: %d",
-				c.origin, len(c.args), c.methodType.NumIn()-1)
-		}
-		if len(c.args) != c.methodType.NumIn() && len(args) != len(c.args) {
-			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: %d",
-				c.origin, len(args), len(c.args))
-		}
-		if len(args) < len(c.args)-1 {
-			return fmt.Errorf("Expected call at %s has the wrong number of arguments. Got: %d, want: greater than or equal to %d",
-				c.origin, len(args), len(c.args)-1)
-		}
-
-		for i, m := range c.args {
-			if i < c.methodType.NumIn()-1 {
-				// Non-variadic args
-				if !m.Matches(args[i]) {
-					return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
-						c.origin, strconv.Itoa(i), args[i], m)
-				}
-				continue
-			}
-			// The last arg has a possibility of a variadic argument, so let it branch
-
-			// sample: Foo(a int, b int, c ...int)
-			if i < len(c.args) && i < len(args) {
-				if m.Matches(args[i]) {
-					// Got Foo(a, b, c) want Foo(matcherA, matcherB, gomock.Any())
-					// Got Foo(a, b, c) want Foo(matcherA, matcherB, someSliceMatcher)
-					// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC)
-					// Got Foo(a, b) want Foo(matcherA, matcherB)
-					// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD)
-					continue
-				}
-			}
-
-			// The number of actual args don't match the number of matchers,
-			// or the last matcher is a slice and the last arg is not.
-			// If this function still matches it is because the last matcher
-			// matches all the remaining arguments or the lack of any.
-			// Convert the remaining arguments, if any, into a slice of the
-			// expected type.
-			vargsType := c.methodType.In(c.methodType.NumIn() - 1)
-			vargs := reflect.MakeSlice(vargsType, 0, len(args)-i)
-			for _, arg := range args[i:] {
-				vargs = reflect.Append(vargs, reflect.ValueOf(arg))
-			}
-			if m.Matches(vargs.Interface()) {
-				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, gomock.Any())
-				// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, someSliceMatcher)
-				// Got Foo(a, b) want Foo(matcherA, matcherB, gomock.Any())
-				// Got Foo(a, b) want Foo(matcherA, matcherB, someEmptySliceMatcher)
-				break
-			}
-			// Wrong number of matchers or not match. Fail.
-			// Got Foo(a, b) want Foo(matcherA, matcherB, matcherC, matcherD)
-			// Got Foo(a, b, c) want Foo(matcherA, matcherB, matcherC, matcherD)
-			// Got Foo(a, b, c, d) want Foo(matcherA, matcherB, matcherC, matcherD, matcherE)
-			// Got Foo(a, b, c, d, e) want Foo(matcherA, matcherB, matcherC, matcherD)
-			// Got Foo(a, b, c) want Foo(matcherA, matcherB)
-			return fmt.Errorf("Expected call at %s doesn't match the argument at index %s.\nGot: %v\nWant: %v",
-				c.origin, strconv.Itoa(i), args[i:], c.args[i])
-
-		}
-	}
-
-	// Check that all prerequisite calls have been satisfied.
-	for _, preReqCall := range c.preReqs {
-		if !preReqCall.satisfied() {
-			return fmt.Errorf("Expected call at %s doesn't have a prerequisite call satisfied:\n%v\nshould be called before:\n%v",
-				c.origin, preReqCall, c)
-		}
-	}
-
-	// Check that the call is not exhausted.
-	if c.exhausted() {
-		return fmt.Errorf("Expected call at %s has already been called the max number of times.", c.origin)
-	}
-
-	return nil
-}
-
-// dropPrereqs tells the expected Call to not re-check prerequisite calls any
-// longer, and to return its current set.
-func (c *Call) dropPrereqs() (preReqs []*Call) {
-	preReqs = c.preReqs
-	c.preReqs = nil
-	return
-}
-
-func (c *Call) call(args []interface{}) []func([]interface{}) []interface{} {
-	c.numCalls++
-	return c.actions
-}
-
-// InOrder declares that the given calls should occur in order.
-func InOrder(calls ...*Call) {
-	for i := 1; i < len(calls); i++ {
-		calls[i].After(calls[i-1])
-	}
-}
-
-func setSlice(arg interface{}, v reflect.Value) {
-	va := reflect.ValueOf(arg)
-	for i := 0; i < v.Len(); i++ {
-		va.Index(i).Set(v.Index(i))
-	}
-}
-
-func (c *Call) addAction(action func([]interface{}) []interface{}) {
-	c.actions = append(c.actions, action)
-}

backend/vendor/github.com/golang/mock/gomock/callset.go

@@ -1,108 +0,0 @@
-// Copyright 2011 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gomock
-
-import (
-	"bytes"
-	"fmt"
-)
-
-// callSet represents a set of expected calls, indexed by receiver and method
-// name.
-type callSet struct {
-	// Calls that are still expected.
-	expected map[callSetKey][]*Call
-	// Calls that have been exhausted.
-	exhausted map[callSetKey][]*Call
-}
-
-// callSetKey is the key in the maps in callSet
-type callSetKey struct {
-	receiver interface{}
-	fname    string
-}
-
-func newCallSet() *callSet {
-	return &callSet{make(map[callSetKey][]*Call), make(map[callSetKey][]*Call)}
-}
-
-// Add adds a new expected call.
-func (cs callSet) Add(call *Call) {
-	key := callSetKey{call.receiver, call.method}
-	m := cs.expected
-	if call.exhausted() {
-		m = cs.exhausted
-	}
-	m[key] = append(m[key], call)
-}
-
-// Remove removes an expected call.
-func (cs callSet) Remove(call *Call) {
-	key := callSetKey{call.receiver, call.method}
-	calls := cs.expected[key]
-	for i, c := range calls {
-		if c == call {
-			// maintain order for remaining calls
-			cs.expected[key] = append(calls[:i], calls[i+1:]...)
-			cs.exhausted[key] = append(cs.exhausted[key], call)
-			break
-		}
-	}
-}
-
-// FindMatch searches for a matching call. Returns error with explanation message if no call matched.
-func (cs callSet) FindMatch(receiver interface{}, method string, args []interface{}) (*Call, error) {
-	key := callSetKey{receiver, method}
-
-	// Search through the expected calls.
-	expected := cs.expected[key]
-	var callsErrors bytes.Buffer
-	for _, call := range expected {
-		err := call.matches(args)
-		if err != nil {
-			fmt.Fprintf(&callsErrors, "\n%v", err)
-		} else {
-			return call, nil
-		}
-	}
-
-	// If we haven't found a match then search through the exhausted calls so we
-	// get useful error messages.
-	exhausted := cs.exhausted[key]
-	for _, call := range exhausted {
-		if err := call.matches(args); err != nil {
-			fmt.Fprintf(&callsErrors, "\n%v", err)
-		}
-	}
-
-	if len(expected)+len(exhausted) == 0 {
-		fmt.Fprintf(&callsErrors, "there are no expected calls of the method %q for that receiver", method)
-	}
-
-	return nil, fmt.Errorf(callsErrors.String())
-}
-
-// Failures returns the calls that are not satisfied.
-func (cs callSet) Failures() []*Call {
-	failures := make([]*Call, 0, len(cs.expected))
-	for _, calls := range cs.expected {
-		for _, call := range calls {
-			if !call.satisfied() {
-				failures = append(failures, call)
-			}
-		}
-	}
-	return failures
-}

backend/vendor/github.com/golang/mock/gomock/controller.go

@@ -1,235 +0,0 @@
-// Copyright 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// GoMock - a mock framework for Go.
-//
-// Standard usage:
-//   (1) Define an interface that you wish to mock.
-//         type MyInterface interface {
-//           SomeMethod(x int64, y string)
-//         }
-//   (2) Use mockgen to generate a mock from the interface.
-//   (3) Use the mock in a test:
-//         func TestMyThing(t *testing.T) {
-//           mockCtrl := gomock.NewController(t)
-//           defer mockCtrl.Finish()
-//
-//           mockObj := something.NewMockMyInterface(mockCtrl)
-//           mockObj.EXPECT().SomeMethod(4, "blah")
-//           // pass mockObj to a real object and play with it.
-//         }
-//
-// By default, expected calls are not enforced to run in any particular order.
-// Call order dependency can be enforced by use of InOrder and/or Call.After.
-// Call.After can create more varied call order dependencies, but InOrder is
-// often more convenient.
-//
-// The following examples create equivalent call order dependencies.
-//
-// Example of using Call.After to chain expected call order:
-//
-//     firstCall := mockObj.EXPECT().SomeMethod(1, "first")
-//     secondCall := mockObj.EXPECT().SomeMethod(2, "second").After(firstCall)
-//     mockObj.EXPECT().SomeMethod(3, "third").After(secondCall)
-//
-// Example of using InOrder to declare expected call order:
-//
-//     gomock.InOrder(
-//         mockObj.EXPECT().SomeMethod(1, "first"),
-//         mockObj.EXPECT().SomeMethod(2, "second"),
-//         mockObj.EXPECT().SomeMethod(3, "third"),
-//     )
-//
-// TODO:
-//	- Handle different argument/return types (e.g. ..., chan, map, interface).
-package gomock
-
-import (
-	"context"
-	"fmt"
-	"reflect"
-	"runtime"
-	"sync"
-)
-
-// A TestReporter is something that can be used to report test failures.
-// It is satisfied by the standard library's *testing.T.
-type TestReporter interface {
-	Errorf(format string, args ...interface{})
-	Fatalf(format string, args ...interface{})
-}
-
-// TestHelper is a TestReporter that has the Helper method.  It is satisfied
-// by the standard library's *testing.T.
-type TestHelper interface {
-	TestReporter
-	Helper()
-}
-
-// A Controller represents the top-level control of a mock ecosystem.
-// It defines the scope and lifetime of mock objects, as well as their expectations.
-// It is safe to call Controller's methods from multiple goroutines.
-type Controller struct {
-	// T should only be called within a generated mock. It is not intended to
-	// be used in user code and may be changed in future versions. T is the
-	// TestReporter passed in when creating the Controller via NewController.
-	// If the TestReporter does not implment a TestHelper it will be wrapped
-	// with a nopTestHelper.
-	T             TestHelper
-	mu            sync.Mutex
-	expectedCalls *callSet
-	finished      bool
-}
-
-func NewController(t TestReporter) *Controller {
-	h, ok := t.(TestHelper)
-	if !ok {
-		h = nopTestHelper{t}
-	}
-
-	return &Controller{
-		T:             h,
-		expectedCalls: newCallSet(),
-	}
-}
-
-type cancelReporter struct {
-	TestHelper
-	cancel func()
-}
-
-func (r *cancelReporter) Errorf(format string, args ...interface{}) {
-	r.TestHelper.Errorf(format, args...)
-}
-func (r *cancelReporter) Fatalf(format string, args ...interface{}) {
-	defer r.cancel()
-	r.TestHelper.Fatalf(format, args...)
-}
-
-// WithContext returns a new Controller and a Context, which is cancelled on any
-// fatal failure.
-func WithContext(ctx context.Context, t TestReporter) (*Controller, context.Context) {
-	h, ok := t.(TestHelper)
-	if !ok {
-		h = nopTestHelper{t}
-	}
-
-	ctx, cancel := context.WithCancel(ctx)
-	return NewController(&cancelReporter{h, cancel}), ctx
-}
-
-type nopTestHelper struct {
-	TestReporter
-}
-
-func (h nopTestHelper) Helper() {}
-
-func (ctrl *Controller) RecordCall(receiver interface{}, method string, args ...interface{}) *Call {
-	ctrl.T.Helper()
-
-	recv := reflect.ValueOf(receiver)
-	for i := 0; i < recv.Type().NumMethod(); i++ {
-		if recv.Type().Method(i).Name == method {
-			return ctrl.RecordCallWithMethodType(receiver, method, recv.Method(i).Type(), args...)
-		}
-	}
-	ctrl.T.Fatalf("gomock: failed finding method %s on %T", method, receiver)
-	panic("unreachable")
-}
-
-func (ctrl *Controller) RecordCallWithMethodType(receiver interface{}, method string, methodType reflect.Type, args ...interface{}) *Call {
-	ctrl.T.Helper()
-
-	call := newCall(ctrl.T, receiver, method, methodType, args...)
-
-	ctrl.mu.Lock()
-	defer ctrl.mu.Unlock()
-	ctrl.expectedCalls.Add(call)
-
-	return call
-}
-
-func (ctrl *Controller) Call(receiver interface{}, method string, args ...interface{}) []interface{} {
-	ctrl.T.Helper()
-
-	// Nest this code so we can use defer to make sure the lock is released.
-	actions := func() []func([]interface{}) []interface{} {
-		ctrl.T.Helper()
-		ctrl.mu.Lock()
-		defer ctrl.mu.Unlock()
-
-		expected, err := ctrl.expectedCalls.FindMatch(receiver, method, args)
-		if err != nil {
-			origin := callerInfo(2)
-			ctrl.T.Fatalf("Unexpected call to %T.%v(%v) at %s because: %s", receiver, method, args, origin, err)
-		}
-
-		// Two things happen here:
-		// * the matching call no longer needs to check prerequite calls,
-		// * and the prerequite calls are no longer expected, so remove them.
-		preReqCalls := expected.dropPrereqs()
-		for _, preReqCall := range preReqCalls {
-			ctrl.expectedCalls.Remove(preReqCall)
-		}
-
-		actions := expected.call(args)
-		if expected.exhausted() {
-			ctrl.expectedCalls.Remove(expected)
-		}
-		return actions
-	}()
-
-	var rets []interface{}
-	for _, action := range actions {
-		if r := action(args); r != nil {
-			rets = r
-		}
-	}
-
-	return rets
-}
-
-func (ctrl *Controller) Finish() {
-	ctrl.T.Helper()
-
-	ctrl.mu.Lock()
-	defer ctrl.mu.Unlock()
-
-	if ctrl.finished {
-		ctrl.T.Fatalf("Controller.Finish was called more than once. It has to be called exactly once.")
-	}
-	ctrl.finished = true
-
-	// If we're currently panicking, probably because this is a deferred call,
-	// pass through the panic.
-	if err := recover(); err != nil {
-		panic(err)
-	}
-
-	// Check that all remaining expected calls are satisfied.
-	failures := ctrl.expectedCalls.Failures()
-	for _, call := range failures {
-		ctrl.T.Errorf("missing call(s) to %v", call)
-	}
-	if len(failures) != 0 {
-		ctrl.T.Fatalf("aborting test due to missing call(s)")
-	}
-}
-
-func callerInfo(skip int) string {
-	if _, file, line, ok := runtime.Caller(skip + 1); ok {
-		return fmt.Sprintf("%s:%d", file, line)
-	}
-	return "unknown file"
-}

backend/vendor/github.com/golang/mock/gomock/matchers.go

@@ -1,122 +0,0 @@
-// Copyright 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package gomock
-
-import (
-	"fmt"
-	"reflect"
-)
-
-// A Matcher is a representation of a class of values.
-// It is used to represent the valid or expected arguments to a mocked method.
-type Matcher interface {
-	// Matches returns whether x is a match.
-	Matches(x interface{}) bool
-
-	// String describes what the matcher matches.
-	String() string
-}
-
-type anyMatcher struct{}
-
-func (anyMatcher) Matches(x interface{}) bool {
-	return true
-}
-
-func (anyMatcher) String() string {
-	return "is anything"
-}
-
-type eqMatcher struct {
-	x interface{}
-}
-
-func (e eqMatcher) Matches(x interface{}) bool {
-	return reflect.DeepEqual(e.x, x)
-}
-
-func (e eqMatcher) String() string {
-	return fmt.Sprintf("is equal to %v", e.x)
-}
-
-type nilMatcher struct{}
-
-func (nilMatcher) Matches(x interface{}) bool {
-	if x == nil {
-		return true
-	}
-
-	v := reflect.ValueOf(x)
-	switch v.Kind() {
-	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map,
-		reflect.Ptr, reflect.Slice:
-		return v.IsNil()
-	}
-
-	return false
-}
-
-func (nilMatcher) String() string {
-	return "is nil"
-}
-
-type notMatcher struct {
-	m Matcher
-}
-
-func (n notMatcher) Matches(x interface{}) bool {
-	return !n.m.Matches(x)
-}
-
-func (n notMatcher) String() string {
-	// TODO: Improve this if we add a NotString method to the Matcher interface.
-	return "not(" + n.m.String() + ")"
-}
-
-type assignableToTypeOfMatcher struct {
-	targetType reflect.Type
-}
-
-func (m assignableToTypeOfMatcher) Matches(x interface{}) bool {
-	return reflect.TypeOf(x).AssignableTo(m.targetType)
-}
-
-func (m assignableToTypeOfMatcher) String() string {
-	return "is assignable to " + m.targetType.Name()
-}
-
-// Constructors
-func Any() Matcher             { return anyMatcher{} }
-func Eq(x interface{}) Matcher { return eqMatcher{x} }
-func Nil() Matcher             { return nilMatcher{} }
-func Not(x interface{}) Matcher {
-	if m, ok := x.(Matcher); ok {
-		return notMatcher{m}
-	}
-	return notMatcher{Eq(x)}
-}
-
-// AssignableToTypeOf is a Matcher that matches if the parameter to the mock
-// function is assignable to the type of the parameter to this function.
-//
-// Example usage:
-//
-// 		dbMock.EXPECT().
-// 			Insert(gomock.AssignableToTypeOf(&EmployeeRecord{})).
-// 			Return(errors.New("DB error"))
-//
-func AssignableToTypeOf(x interface{}) Matcher {
-	return assignableToTypeOfMatcher{reflect.TypeOf(x)}
-}

backend/vendor/github.com/golang/protobuf/proto/decode.go

@@ -186,7 +186,6 @@ func (p *Buffer) DecodeVarint() (x uint64, err error) {
 	if b&0x80 == 0 {
 		goto done
 	}
-	// x -= 0x80 << 63 // Always zero.
 
 	return 0, errOverflow
 

backend/vendor/github.com/golang/protobuf/proto/deprecated.go

@@ -0,0 +1,63 @@
+// Go support for Protocol Buffers - Google's data interchange format
+//
+// Copyright 2018 The Go Authors.  All rights reserved.
+// https://github.com/golang/protobuf
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package proto
+
+import "errors"
+
+// Deprecated: do not use.
+type Stats struct{ Emalloc, Dmalloc, Encode, Decode, Chit, Cmiss, Size uint64 }
+
+// Deprecated: do not use.
+func GetStats() Stats { return Stats{} }
+
+// Deprecated: do not use.
+func MarshalMessageSet(interface{}) ([]byte, error) {
+	return nil, errors.New("proto: not implemented")
+}
+
+// Deprecated: do not use.
+func UnmarshalMessageSet([]byte, interface{}) error {
+	return errors.New("proto: not implemented")
+}
+
+// Deprecated: do not use.
+func MarshalMessageSetJSON(interface{}) ([]byte, error) {
+	return nil, errors.New("proto: not implemented")
+}
+
+// Deprecated: do not use.
+func UnmarshalMessageSetJSON([]byte, interface{}) error {
+	return errors.New("proto: not implemented")
+}
+
+// Deprecated: do not use.
+func RegisterMessageSetType(Message, int32, string) {}

backend/vendor/github.com/golang/protobuf/proto/equal.go

@@ -246,7 +246,8 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool {
 			return false
 		}
 
-		m1, m2 := e1.value, e2.value
+		m1 := extensionAsLegacyType(e1.value)
+		m2 := extensionAsLegacyType(e2.value)
 
 		if m1 == nil && m2 == nil {
 			// Both have only encoded form.

backend/vendor/github.com/golang/protobuf/proto/extensions.go

@@ -185,9 +185,25 @@ type Extension struct {
 	// extension will have only enc set. When such an extension is
 	// accessed using GetExtension (or GetExtensions) desc and value
 	// will be set.
-	desc  *ExtensionDesc
+	desc *ExtensionDesc
+
+	// value is a concrete value for the extension field. Let the type of
+	// desc.ExtensionType be the "API type" and the type of Extension.value
+	// be the "storage type". The API type and storage type are the same except:
+	//	* For scalars (except []byte), the API type uses *T,
+	//	while the storage type uses T.
+	//	* For repeated fields, the API type uses []T, while the storage type
+	//	uses *[]T.
+	//
+	// The reason for the divergence is so that the storage type more naturally
+	// matches what is expected of when retrieving the values through the
+	// protobuf reflection APIs.
+	//
+	// The value may only be populated if desc is also populated.
 	value interface{}
-	enc   []byte
+
+	// enc is the raw bytes for the extension field.
+	enc []byte
 }
 
 // SetRawExtension is for testing only.
@@ -334,7 +350,7 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) {
 			// descriptors with the same field number.
 			return nil, errors.New("proto: descriptor conflict")
 		}
-		return e.value, nil
+		return extensionAsLegacyType(e.value), nil
 	}
 
 	if extension.ExtensionType == nil {
@@ -349,11 +365,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) {
 
 	// Remember the decoded version and drop the encoded version.
 	// That way it is safe to mutate what we return.
-	e.value = v
+	e.value = extensionAsStorageType(v)
 	e.desc = extension
 	e.enc = nil
 	emap[extension.Field] = e
-	return e.value, nil
+	return extensionAsLegacyType(e.value), nil
 }
 
 // defaultExtensionValue returns the default value for extension.
@@ -488,7 +504,7 @@ func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error
 	}
 	typ := reflect.TypeOf(extension.ExtensionType)
 	if typ != reflect.TypeOf(value) {
-		return errors.New("proto: bad extension value type")
+		return fmt.Errorf("proto: bad extension value type. got: %T, want: %T", value, extension.ExtensionType)
 	}
 	// nil extension values need to be caught early, because the
 	// encoder can't distinguish an ErrNil due to a nil extension
@@ -500,7 +516,7 @@ func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error
 	}
 
 	extmap := epb.extensionsWrite()
-	extmap[extension.Field] = Extension{desc: extension, value: value}
+	extmap[extension.Field] = Extension{desc: extension, value: extensionAsStorageType(value)}
 	return nil
 }
 
@@ -541,3 +557,51 @@ func RegisterExtension(desc *ExtensionDesc) {
 func RegisteredExtensions(pb Message) map[int32]*ExtensionDesc {
 	return extensionMaps[reflect.TypeOf(pb).Elem()]
 }
+
+// extensionAsLegacyType converts an value in the storage type as the API type.
+// See Extension.value.
+func extensionAsLegacyType(v interface{}) interface{} {
+	switch rv := reflect.ValueOf(v); rv.Kind() {
+	case reflect.Bool, reflect.Int32, reflect.Int64, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64, reflect.String:
+		// Represent primitive types as a pointer to the value.
+		rv2 := reflect.New(rv.Type())
+		rv2.Elem().Set(rv)
+		v = rv2.Interface()
+	case reflect.Ptr:
+		// Represent slice types as the value itself.
+		switch rv.Type().Elem().Kind() {
+		case reflect.Slice:
+			if rv.IsNil() {
+				v = reflect.Zero(rv.Type().Elem()).Interface()
+			} else {
+				v = rv.Elem().Interface()
+			}
+		}
+	}
+	return v
+}
+
+// extensionAsStorageType converts an value in the API type as the storage type.
+// See Extension.value.
+func extensionAsStorageType(v interface{}) interface{} {
+	switch rv := reflect.ValueOf(v); rv.Kind() {
+	case reflect.Ptr:
+		// Represent slice types as the value itself.
+		switch rv.Type().Elem().Kind() {
+		case reflect.Bool, reflect.Int32, reflect.Int64, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64, reflect.String:
+			if rv.IsNil() {
+				v = reflect.Zero(rv.Type().Elem()).Interface()
+			} else {
+				v = rv.Elem().Interface()
+			}
+		}
+	case reflect.Slice:
+		// Represent slice types as a pointer to the value.
+		if rv.Type().Elem().Kind() != reflect.Uint8 {
+			rv2 := reflect.New(rv.Type())
+			rv2.Elem().Set(rv)
+			v = rv2.Interface()
+		}
+	}
+	return v
+}

backend/vendor/github.com/golang/protobuf/proto/lib.go

@@ -341,26 +341,6 @@ type Message interface {
 	ProtoMessage()
 }
 
-// Stats records allocation details about the protocol buffer encoders
-// and decoders.  Useful for tuning the library itself.
-type Stats struct {
-	Emalloc uint64 // mallocs in encode
-	Dmalloc uint64 // mallocs in decode
-	Encode  uint64 // number of encodes
-	Decode  uint64 // number of decodes
-	Chit    uint64 // number of cache hits
-	Cmiss   uint64 // number of cache misses
-	Size    uint64 // number of sizes
-}
-
-// Set to true to enable stats collection.
-const collectStats = false
-
-var stats Stats
-
-// GetStats returns a copy of the global Stats structure.
-func GetStats() Stats { return stats }
-
 // A Buffer is a buffer manager for marshaling and unmarshaling
 // protocol buffers.  It may be reused between invocations to
 // reduce memory usage.  It is not necessary to use a Buffer;
@@ -960,13 +940,19 @@ func isProto3Zero(v reflect.Value) bool {
 	return false
 }
 
-// ProtoPackageIsVersion2 is referenced from generated protocol buffer files
-// to assert that that code is compatible with this version of the proto package.
-const ProtoPackageIsVersion2 = true
+const (
+	// ProtoPackageIsVersion3 is referenced from generated protocol buffer files
+	// to assert that that code is compatible with this version of the proto package.
+	ProtoPackageIsVersion3 = true
 
-// ProtoPackageIsVersion1 is referenced from generated protocol buffer files
-// to assert that that code is compatible with this version of the proto package.
-const ProtoPackageIsVersion1 = true
+	// ProtoPackageIsVersion2 is referenced from generated protocol buffer files
+	// to assert that that code is compatible with this version of the proto package.
+	ProtoPackageIsVersion2 = true
+
+	// ProtoPackageIsVersion1 is referenced from generated protocol buffer files
+	// to assert that that code is compatible with this version of the proto package.
+	ProtoPackageIsVersion1 = true
+)
 
 // InternalMessageInfo is a type used internally by generated .pb.go files.
 // This type is not intended to be used by non-generated code.

backend/vendor/github.com/golang/protobuf/proto/message_set.go

@@ -36,13 +36,7 @@ package proto
  */
 
 import (
-	"bytes"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"reflect"
-	"sort"
-	"sync"
 )
 
 // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID.
@@ -145,46 +139,9 @@ func skipVarint(buf []byte) []byte {
 	return buf[i+1:]
 }
 
-// MarshalMessageSet encodes the extension map represented by m in the message set wire format.
-// It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option.
-func MarshalMessageSet(exts interface{}) ([]byte, error) {
-	return marshalMessageSet(exts, false)
-}
-
-// marshaMessageSet implements above function, with the opt to turn on / off deterministic during Marshal.
-func marshalMessageSet(exts interface{}, deterministic bool) ([]byte, error) {
-	switch exts := exts.(type) {
-	case *XXX_InternalExtensions:
-		var u marshalInfo
-		siz := u.sizeMessageSet(exts)
-		b := make([]byte, 0, siz)
-		return u.appendMessageSet(b, exts, deterministic)
-
-	case map[int32]Extension:
-		// This is an old-style extension map.
-		// Wrap it in a new-style XXX_InternalExtensions.
-		ie := XXX_InternalExtensions{
-			p: &struct {
-				mu           sync.Mutex
-				extensionMap map[int32]Extension
-			}{
-				extensionMap: exts,
-			},
-		}
-
-		var u marshalInfo
-		siz := u.sizeMessageSet(&ie)
-		b := make([]byte, 0, siz)
-		return u.appendMessageSet(b, &ie, deterministic)
-
-	default:
-		return nil, errors.New("proto: not an extension map")
-	}
-}
-
-// UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format.
+// unmarshalMessageSet decodes the extension map encoded in buf in the message set wire format.
 // It is called by Unmarshal methods on protocol buffer messages with the message_set_wire_format option.
-func UnmarshalMessageSet(buf []byte, exts interface{}) error {
+func unmarshalMessageSet(buf []byte, exts interface{}) error {
 	var m map[int32]Extension
 	switch exts := exts.(type) {
 	case *XXX_InternalExtensions:
@@ -222,93 +179,3 @@ func UnmarshalMessageSet(buf []byte, exts interface{}) error {
 	}
 	return nil
 }
-
-// MarshalMessageSetJSON encodes the extension map represented by m in JSON format.
-// It is called by generated MarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
-func MarshalMessageSetJSON(exts interface{}) ([]byte, error) {
-	var m map[int32]Extension
-	switch exts := exts.(type) {
-	case *XXX_InternalExtensions:
-		var mu sync.Locker
-		m, mu = exts.extensionsRead()
-		if m != nil {
-			// Keep the extensions map locked until we're done marshaling to prevent
-			// races between marshaling and unmarshaling the lazily-{en,de}coded
-			// values.
-			mu.Lock()
-			defer mu.Unlock()
-		}
-	case map[int32]Extension:
-		m = exts
-	default:
-		return nil, errors.New("proto: not an extension map")
-	}
-	var b bytes.Buffer
-	b.WriteByte('{')
-
-	// Process the map in key order for deterministic output.
-	ids := make([]int32, 0, len(m))
-	for id := range m {
-		ids = append(ids, id)
-	}
-	sort.Sort(int32Slice(ids)) // int32Slice defined in text.go
-
-	for i, id := range ids {
-		ext := m[id]
-		msd, ok := messageSetMap[id]
-		if !ok {
-			// Unknown type; we can't render it, so skip it.
-			continue
-		}
-
-		if i > 0 && b.Len() > 1 {
-			b.WriteByte(',')
-		}
-
-		fmt.Fprintf(&b, `"[%s]":`, msd.name)
-
-		x := ext.value
-		if x == nil {
-			x = reflect.New(msd.t.Elem()).Interface()
-			if err := Unmarshal(ext.enc, x.(Message)); err != nil {
-				return nil, err
-			}
-		}
-		d, err := json.Marshal(x)
-		if err != nil {
-			return nil, err
-		}
-		b.Write(d)
-	}
-	b.WriteByte('}')
-	return b.Bytes(), nil
-}
-
-// UnmarshalMessageSetJSON decodes the extension map encoded in buf in JSON format.
-// It is called by generated UnmarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
-func UnmarshalMessageSetJSON(buf []byte, exts interface{}) error {
-	// Common-case fast path.
-	if len(buf) == 0 || bytes.Equal(buf, []byte("{}")) {
-		return nil
-	}
-
-	// This is fairly tricky, and it's not clear that it is needed.
-	return errors.New("TODO: UnmarshalMessageSetJSON not yet implemented")
-}
-
-// A global registry of types that can be used in a MessageSet.
-
-var messageSetMap = make(map[int32]messageSetDesc)
-
-type messageSetDesc struct {
-	t    reflect.Type // pointer to struct
-	name string
-}
-
-// RegisterMessageSetType is called from the generated code.
-func RegisterMessageSetType(m Message, fieldNum int32, name string) {
-	messageSetMap[fieldNum] = messageSetDesc{
-		t:    reflect.TypeOf(m),
-		name: name,
-	}
-}

backend/vendor/github.com/golang/protobuf/proto/pointer_reflect.go

@@ -79,10 +79,13 @@ func toPointer(i *Message) pointer {
 
 // toAddrPointer converts an interface to a pointer that points to
 // the interface data.
-func toAddrPointer(i *interface{}, isptr bool) pointer {
+func toAddrPointer(i *interface{}, isptr, deref bool) pointer {
 	v := reflect.ValueOf(*i)
 	u := reflect.New(v.Type())
 	u.Elem().Set(v)
+	if deref {
+		u = u.Elem()
+	}
 	return pointer{v: u}
 }
 

backend/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go

@@ -85,16 +85,21 @@ func toPointer(i *Message) pointer {
 
 // toAddrPointer converts an interface to a pointer that points to
 // the interface data.
-func toAddrPointer(i *interface{}, isptr bool) pointer {
+func toAddrPointer(i *interface{}, isptr, deref bool) (p pointer) {
 	// Super-tricky - read or get the address of data word of interface value.
 	if isptr {
 		// The interface is of pointer type, thus it is a direct interface.
 		// The data word is the pointer data itself. We take its address.
-		return pointer{p: unsafe.Pointer(uintptr(unsafe.Pointer(i)) + ptrSize)}
+		p = pointer{p: unsafe.Pointer(uintptr(unsafe.Pointer(i)) + ptrSize)}
+	} else {
+		// The interface is not of pointer type. The data word is the pointer
+		// to the data.
+		p = pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]}
 	}
-	// The interface is not of pointer type. The data word is the pointer
-	// to the data.
-	return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]}
+	if deref {
+		p.p = *(*unsafe.Pointer)(p.p)
+	}
+	return p
 }
 
 // valToPointer converts v to a pointer. v must be of pointer type.

backend/vendor/github.com/golang/protobuf/proto/properties.go

@@ -334,9 +334,6 @@ func GetProperties(t reflect.Type) *StructProperties {
 	sprop, ok := propertiesMap[t]
 	propertiesMu.RUnlock()
 	if ok {
-		if collectStats {
-			stats.Chit++
-		}
 		return sprop
 	}
 
@@ -346,17 +343,20 @@ func GetProperties(t reflect.Type) *StructProperties {
 	return sprop
 }
 
+type (
+	oneofFuncsIface interface {
+		XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
+	}
+	oneofWrappersIface interface {
+		XXX_OneofWrappers() []interface{}
+	}
+)
+
 // getPropertiesLocked requires that propertiesMu is held.
 func getPropertiesLocked(t reflect.Type) *StructProperties {
 	if prop, ok := propertiesMap[t]; ok {
-		if collectStats {
-			stats.Chit++
-		}
 		return prop
 	}
-	if collectStats {
-		stats.Cmiss++
-	}
 
 	prop := new(StructProperties)
 	// in case of recursive protos, fill this in now.
@@ -391,13 +391,14 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
 	// Re-order prop.order.
 	sort.Sort(prop)
 
-	type oneofMessage interface {
-		XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
+	var oots []interface{}
+	switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+	case oneofFuncsIface:
+		_, _, _, oots = m.XXX_OneofFuncs()
+	case oneofWrappersIface:
+		oots = m.XXX_OneofWrappers()
 	}
-	if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok {
-		var oots []interface{}
-		_, _, _, oots = om.XXX_OneofFuncs()
-
+	if len(oots) > 0 {
 		// Interpret oneof metadata.
 		prop.OneofTypes = make(map[string]*OneofProperties)
 		for _, oot := range oots {

backend/vendor/github.com/golang/protobuf/proto/table_marshal.go

@@ -87,6 +87,7 @@ type marshalElemInfo struct {
 	sizer     sizer
 	marshaler marshaler
 	isptr     bool // elem is pointer typed, thus interface of this type is a direct interface (extension only)
+	deref     bool // dereference the pointer before operating on it; implies isptr
 }
 
 var (
@@ -320,8 +321,11 @@ func (u *marshalInfo) computeMarshalInfo() {
 
 	// get oneof implementers
 	var oneofImplementers []interface{}
-	if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok {
+	switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+	case oneofFuncsIface:
 		_, _, _, oneofImplementers = m.XXX_OneofFuncs()
+	case oneofWrappersIface:
+		oneofImplementers = m.XXX_OneofWrappers()
 	}
 
 	n := t.NumField()
@@ -407,13 +411,22 @@ func (u *marshalInfo) getExtElemInfo(desc *ExtensionDesc) *marshalElemInfo {
 		panic("tag is not an integer")
 	}
 	wt := wiretype(tags[0])
+	if t.Kind() == reflect.Ptr && t.Elem().Kind() != reflect.Struct {
+		t = t.Elem()
+	}
 	sizer, marshaler := typeMarshaler(t, tags, false, false)
+	var deref bool
+	if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 {
+		t = reflect.PtrTo(t)
+		deref = true
+	}
 	e = &marshalElemInfo{
 		wiretag:   uint64(tag)<<3 | wt,
 		tagsize:   SizeVarint(uint64(tag) << 3),
 		sizer:     sizer,
 		marshaler: marshaler,
 		isptr:     t.Kind() == reflect.Ptr,
+		deref:     deref,
 	}
 
 	// update cache
@@ -448,7 +461,7 @@ func (fi *marshalFieldInfo) computeMarshalFieldInfo(f *reflect.StructField) {
 
 func (fi *marshalFieldInfo) computeOneofFieldInfo(f *reflect.StructField, oneofImplementers []interface{}) {
 	fi.field = toField(f)
-	fi.wiretag = 1<<31 - 1 // Use a large tag number, make oneofs sorted at the end. This tag will not appear on the wire.
+	fi.wiretag = math.MaxInt32 // Use a large tag number, make oneofs sorted at the end. This tag will not appear on the wire.
 	fi.isPointer = true
 	fi.sizer, fi.marshaler = makeOneOfMarshaler(fi, f)
 	fi.oneofElems = make(map[reflect.Type]*marshalElemInfo)
@@ -476,10 +489,6 @@ func (fi *marshalFieldInfo) computeOneofFieldInfo(f *reflect.StructField, oneofI
 	}
 }
 
-type oneofMessage interface {
-	XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
-}
-
 // wiretype returns the wire encoding of the type.
 func wiretype(encoding string) uint64 {
 	switch encoding {
@@ -2310,8 +2319,8 @@ func makeMapMarshaler(f *reflect.StructField) (sizer, marshaler) {
 			for _, k := range m.MapKeys() {
 				ki := k.Interface()
 				vi := m.MapIndex(k).Interface()
-				kaddr := toAddrPointer(&ki, false)             // pointer to key
-				vaddr := toAddrPointer(&vi, valIsPtr)          // pointer to value
+				kaddr := toAddrPointer(&ki, false, false)      // pointer to key
+				vaddr := toAddrPointer(&vi, valIsPtr, false)   // pointer to value
 				siz := keySizer(kaddr, 1) + valSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1)
 				n += siz + SizeVarint(uint64(siz)) + tagsize
 			}
@@ -2329,8 +2338,8 @@ func makeMapMarshaler(f *reflect.StructField) (sizer, marshaler) {
 			for _, k := range keys {
 				ki := k.Interface()
 				vi := m.MapIndex(k).Interface()
-				kaddr := toAddrPointer(&ki, false)    // pointer to key
-				vaddr := toAddrPointer(&vi, valIsPtr) // pointer to value
+				kaddr := toAddrPointer(&ki, false, false)    // pointer to key
+				vaddr := toAddrPointer(&vi, valIsPtr, false) // pointer to value
 				b = appendVarint(b, tag)
 				siz := keySizer(kaddr, 1) + valCachedSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1)
 				b = appendVarint(b, uint64(siz))
@@ -2399,7 +2408,7 @@ func (u *marshalInfo) sizeExtensions(ext *XXX_InternalExtensions) int {
 		// the last time this function was called.
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		n += ei.sizer(p, ei.tagsize)
 	}
 	mu.Unlock()
@@ -2434,7 +2443,7 @@ func (u *marshalInfo) appendExtensions(b []byte, ext *XXX_InternalExtensions, de
 
 			ei := u.getExtElemInfo(e.desc)
 			v := e.value
-			p := toAddrPointer(&v, ei.isptr)
+			p := toAddrPointer(&v, ei.isptr, ei.deref)
 			b, err = ei.marshaler(b, p, ei.wiretag, deterministic)
 			if !nerr.Merge(err) {
 				return b, err
@@ -2465,7 +2474,7 @@ func (u *marshalInfo) appendExtensions(b []byte, ext *XXX_InternalExtensions, de
 
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		b, err = ei.marshaler(b, p, ei.wiretag, deterministic)
 		if !nerr.Merge(err) {
 			return b, err
@@ -2510,7 +2519,7 @@ func (u *marshalInfo) sizeMessageSet(ext *XXX_InternalExtensions) int {
 
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		n += ei.sizer(p, 1) // message, tag = 3 (size=1)
 	}
 	mu.Unlock()
@@ -2553,7 +2562,7 @@ func (u *marshalInfo) appendMessageSet(b []byte, ext *XXX_InternalExtensions, de
 
 			ei := u.getExtElemInfo(e.desc)
 			v := e.value
-			p := toAddrPointer(&v, ei.isptr)
+			p := toAddrPointer(&v, ei.isptr, ei.deref)
 			b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic)
 			if !nerr.Merge(err) {
 				return b, err
@@ -2591,7 +2600,7 @@ func (u *marshalInfo) appendMessageSet(b []byte, ext *XXX_InternalExtensions, de
 
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic)
 		b = append(b, 1<<3|WireEndGroup)
 		if !nerr.Merge(err) {
@@ -2621,7 +2630,7 @@ func (u *marshalInfo) sizeV1Extensions(m map[int32]Extension) int {
 
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		n += ei.sizer(p, ei.tagsize)
 	}
 	return n
@@ -2656,7 +2665,7 @@ func (u *marshalInfo) appendV1Extensions(b []byte, m map[int32]Extension, determ
 
 		ei := u.getExtElemInfo(e.desc)
 		v := e.value
-		p := toAddrPointer(&v, ei.isptr)
+		p := toAddrPointer(&v, ei.isptr, ei.deref)
 		b, err = ei.marshaler(b, p, ei.wiretag, deterministic)
 		if !nerr.Merge(err) {
 			return b, err

backend/vendor/github.com/golang/protobuf/proto/table_unmarshal.go

@@ -136,7 +136,7 @@ func (u *unmarshalInfo) unmarshal(m pointer, b []byte) error {
 		u.computeUnmarshalInfo()
 	}
 	if u.isMessageSet {
-		return UnmarshalMessageSet(b, m.offset(u.extensions).toExtensions())
+		return unmarshalMessageSet(b, m.offset(u.extensions).toExtensions())
 	}
 	var reqMask uint64 // bitmask of required fields we've seen.
 	var errLater error
@@ -362,46 +362,48 @@ func (u *unmarshalInfo) computeUnmarshalInfo() {
 	}
 
 	// Find any types associated with oneof fields.
-	// TODO: XXX_OneofFuncs returns more info than we need.  Get rid of some of it?
-	fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("XXX_OneofFuncs")
-	if fn.IsValid() {
-		res := fn.Call(nil)[3] // last return value from XXX_OneofFuncs: []interface{}
-		for i := res.Len() - 1; i >= 0; i-- {
-			v := res.Index(i)                             // interface{}
-			tptr := reflect.ValueOf(v.Interface()).Type() // *Msg_X
-			typ := tptr.Elem()                            // Msg_X
+	var oneofImplementers []interface{}
+	switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+	case oneofFuncsIface:
+		_, _, _, oneofImplementers = m.XXX_OneofFuncs()
+	case oneofWrappersIface:
+		oneofImplementers = m.XXX_OneofWrappers()
+	}
+	for _, v := range oneofImplementers {
+		tptr := reflect.TypeOf(v) // *Msg_X
+		typ := tptr.Elem()        // Msg_X
 
-			f := typ.Field(0) // oneof implementers have one field
-			baseUnmarshal := fieldUnmarshaler(&f)
-			tags := strings.Split(f.Tag.Get("protobuf"), ",")
-			fieldNum, err := strconv.Atoi(tags[1])
-			if err != nil {
-				panic("protobuf tag field not an integer: " + tags[1])
-			}
-			var name string
-			for _, tag := range tags {
-				if strings.HasPrefix(tag, "name=") {
-					name = strings.TrimPrefix(tag, "name=")
-					break
-				}
-			}
-
-			// Find the oneof field that this struct implements.
-			// Might take O(n^2) to process all of the oneofs, but who cares.
-			for _, of := range oneofFields {
-				if tptr.Implements(of.ityp) {
-					// We have found the corresponding interface for this struct.
-					// That lets us know where this struct should be stored
-					// when we encounter it during unmarshaling.
-					unmarshal := makeUnmarshalOneof(typ, of.ityp, baseUnmarshal)
-					u.setTag(fieldNum, of.field, unmarshal, 0, name)
-				}
+		f := typ.Field(0) // oneof implementers have one field
+		baseUnmarshal := fieldUnmarshaler(&f)
+		tags := strings.Split(f.Tag.Get("protobuf"), ",")
+		fieldNum, err := strconv.Atoi(tags[1])
+		if err != nil {
+			panic("protobuf tag field not an integer: " + tags[1])
+		}
+		var name string
+		for _, tag := range tags {
+			if strings.HasPrefix(tag, "name=") {
+				name = strings.TrimPrefix(tag, "name=")
+				break
 			}
 		}
+
+		// Find the oneof field that this struct implements.
+		// Might take O(n^2) to process all of the oneofs, but who cares.
+		for _, of := range oneofFields {
+			if tptr.Implements(of.ityp) {
+				// We have found the corresponding interface for this struct.
+				// That lets us know where this struct should be stored
+				// when we encounter it during unmarshaling.
+				unmarshal := makeUnmarshalOneof(typ, of.ityp, baseUnmarshal)
+				u.setTag(fieldNum, of.field, unmarshal, 0, name)
+			}
+		}
+
 	}
 
 	// Get extension ranges, if any.
-	fn = reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray")
+	fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray")
 	if fn.IsValid() {
 		if !u.extensions.IsValid() && !u.oldExtensions.IsValid() {
 			panic("a message with extensions, but no extensions field in " + t.Name())
@@ -1948,7 +1950,7 @@ func encodeVarint(b []byte, x uint64) []byte {
 // If there is an error, it returns 0,0.
 func decodeVarint(b []byte) (uint64, int) {
 	var x, y uint64
-	if len(b) <= 0 {
+	if len(b) == 0 {
 		goto bad
 	}
 	x = uint64(b[0])

backend/vendor/github.com/hashicorp/golang-lru/lru.go

@@ -40,31 +40,35 @@ func (c *Cache) Purge() {
 // Add adds a value to the cache.  Returns true if an eviction occurred.
 func (c *Cache) Add(key, value interface{}) (evicted bool) {
 	c.lock.Lock()
-	defer c.lock.Unlock()
-	return c.lru.Add(key, value)
+	evicted = c.lru.Add(key, value)
+	c.lock.Unlock()
+	return evicted
 }
 
 // Get looks up a key's value from the cache.
 func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
 	c.lock.Lock()
-	defer c.lock.Unlock()
-	return c.lru.Get(key)
+	value, ok = c.lru.Get(key)
+	c.lock.Unlock()
+	return value, ok
 }
 
 // Contains checks if a key is in the cache, without updating the
 // recent-ness or deleting it for being stale.
 func (c *Cache) Contains(key interface{}) bool {
 	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.lru.Contains(key)
+	containKey := c.lru.Contains(key)
+	c.lock.RUnlock()
+	return containKey
 }
 
 // Peek returns the key value (or undefined if not found) without updating
 // the "recently used"-ness of the key.
 func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) {
 	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.lru.Peek(key)
+	value, ok = c.lru.Peek(key)
+	c.lock.RUnlock()
+	return value, ok
 }
 
 // ContainsOrAdd checks if a key is in the cache  without updating the
@@ -98,13 +102,15 @@ func (c *Cache) RemoveOldest() {
 // Keys returns a slice of the keys in the cache, from oldest to newest.
 func (c *Cache) Keys() []interface{} {
 	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.lru.Keys()
+	keys := c.lru.Keys()
+	c.lock.RUnlock()
+	return keys
 }
 
 // Len returns the number of items in the cache.
 func (c *Cache) Len() int {
 	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.lru.Len()
+	length := c.lru.Len()
+	c.lock.RUnlock()
+	return length
 }

backend/vendor/github.com/stretchr/objx/.codeclimate.yml

@@ -10,4 +10,12 @@ exclude_patterns:
 - ".github/"
 - "vendor/"
 - "codegen/"
+- "*.yml"
+- ".*.yml"
+- "*.md"
+- "Gopkg.*"
 - "doc.go"
+- "type_specific_codegen_test.go"
+- "type_specific_codegen.go"
+- ".gitignore"
+- "LICENSE"

backend/vendor/github.com/stretchr/objx/.travis.yml

@@ -1,8 +1,14 @@
 language: go
 go:
-  - 1.8
-  - 1.9
-  - tip
+  - "1.10.x"
+  - "1.11.x"
+  - "1.12.x"
+  - master
+
+matrix:
+  allow_failures:
+    - go: master
+fast_finish: true
 
 env:
   global:
@@ -14,12 +20,11 @@ before_script:
   - ./cc-test-reporter before-build
 
 install:
-- go get github.com/go-task/task/cmd/task
+  - curl -sL https://taskfile.dev/install.sh | sh
 
 script:
-- task dl-deps
-- task lint
-- task test-coverage
+  - diff -u <(echo -n) <(./bin/task lint)
+  - ./bin/task test-coverage
 
 after_script:
   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT

backend/vendor/github.com/stretchr/objx/Gopkg.lock

@@ -1,30 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require"
-  ]
-  revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
-  version = "v1.2.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "2d160a7dea4ffd13c6c31dab40373822f9d78c73beba016d662bef8f7a998876"
-  solver-name = "gps-cdcl"
-  solver-version = 1

backend/vendor/github.com/stretchr/objx/Gopkg.toml

@@ -1,8 +0,0 @@
-[prune]
-  unused-packages = true
-  non-go = true
-  go-tests = true
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "~1.2.0"

backend/vendor/github.com/stretchr/objx/README.md

@@ -74,7 +74,7 @@ To update Objx to the latest version, run:
     go get -u github.com/stretchr/objx
 
 ### Supported go versions
-We support the lastest two major Go versions, which are 1.8 and 1.9 at the moment.
+We support the lastest three major Go versions, which are 1.10, 1.11 and 1.12 at the moment.
 
 ## Contributing
 Please feel free to submit issues, fork the repository and send pull requests!

backend/vendor/github.com/stretchr/objx/Taskfile.yml

@@ -1,32 +1,30 @@
-default:
-  deps: [test]
+version: '2'
 
-dl-deps:
-  desc: Downloads cli dependencies
-  cmds:
-    - go get -u github.com/golang/lint/golint
-    - go get -u github.com/golang/dep/cmd/dep
+env:
+  GOFLAGS: -mod=vendor
 
-update-deps:
-  desc: Updates dependencies
-  cmds:
-    - dep ensure
-    - dep ensure -update
+tasks:
+  default:
+    deps: [test]
 
-lint:
-  desc: Runs golint
-  cmds:
-    - go fmt $(go list ./... | grep -v /vendor/)
-    - go vet $(go list ./... | grep -v /vendor/)
-    - golint $(ls *.go | grep -v "doc.go")
-  silent: true
+  lint:
+    desc: Checks code style
+    cmds:
+      - gofmt -d -s *.go
+      - go vet ./...
+    silent: true
 
-test:
-  desc: Runs go tests
-  cmds:
-    - go test -race  .
+  lint-fix:
+    desc: Fixes code style
+    cmds:
+      - gofmt -w -s *.go
 
-test-coverage:
-  desc: Runs go tests and calucates test coverage
-  cmds:
-    - go test -coverprofile=c.out .
+  test:
+    desc: Runs go tests
+    cmds:
+      - go test -race  ./...
+
+  test-coverage:
+    desc: Runs go tests and calucates test coverage
+    cmds:
+      - go test -race -coverprofile=c.out ./...

backend/vendor/github.com/stretchr/objx/accessors.go

@@ -6,9 +6,17 @@ import (
 	"strings"
 )
 
-// arrayAccesRegexString is the regex used to extract the array number
-// from the access path
-const arrayAccesRegexString = `^(.+)\[([0-9]+)\]$`
+const (
+	// PathSeparator is the character used to separate the elements
+	// of the keypath.
+	//
+	// For example, `location.address.city`
+	PathSeparator string = "."
+
+	// arrayAccesRegexString is the regex used to extract the array number
+	// from the access path
+	arrayAccesRegexString = `^(.+)\[([0-9]+)\]$`
+)
 
 // arrayAccesRegex is the compiled arrayAccesRegexString
 var arrayAccesRegex = regexp.MustCompile(arrayAccesRegexString)
@@ -46,103 +54,66 @@ func (m Map) Set(selector string, value interface{}) Map {
 	return m
 }
 
+// getIndex returns the index, which is hold in s by two braches.
+// It also returns s withour the index part, e.g. name[1] will return (1, name).
+// If no index is found, -1 is returned
+func getIndex(s string) (int, string) {
+	arrayMatches := arrayAccesRegex.FindStringSubmatch(s)
+	if len(arrayMatches) > 0 {
+		// Get the key into the map
+		selector := arrayMatches[1]
+		// Get the index into the array at the key
+		// We know this cannt fail because arrayMatches[2] is an int for sure
+		index, _ := strconv.Atoi(arrayMatches[2])
+		return index, selector
+	}
+	return -1, s
+}
+
 // access accesses the object using the selector and performs the
 // appropriate action.
-func access(current, selector, value interface{}, isSet bool) interface{} {
-	switch selector.(type) {
-	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
+func access(current interface{}, selector string, value interface{}, isSet bool) interface{} {
+	selSegs := strings.SplitN(selector, PathSeparator, 2)
+	thisSel := selSegs[0]
+	index := -1
+
+	if strings.Contains(thisSel, "[") {
+		index, thisSel = getIndex(thisSel)
+	}
+
+	if curMap, ok := current.(Map); ok {
+		current = map[string]interface{}(curMap)
+	}
+	// get the object in question
+	switch current.(type) {
+	case map[string]interface{}:
+		curMSI := current.(map[string]interface{})
+		if len(selSegs) <= 1 && isSet {
+			curMSI[thisSel] = value
+			return nil
+		}
+
+		_, ok := curMSI[thisSel].(map[string]interface{})
+		if (curMSI[thisSel] == nil || !ok) && index == -1 && isSet {
+			curMSI[thisSel] = map[string]interface{}{}
+		}
+
+		current = curMSI[thisSel]
+	default:
+		current = nil
+	}
+	// do we need to access the item of an array?
+	if index > -1 {
 		if array, ok := current.([]interface{}); ok {
-			index := intFromInterface(selector)
-			if index >= len(array) {
-				return nil
-			}
-			return array[index]
-		}
-		return nil
-
-	case string:
-		selStr := selector.(string)
-		selSegs := strings.SplitN(selStr, PathSeparator, 2)
-		thisSel := selSegs[0]
-		index := -1
-		var err error
-
-		if strings.Contains(thisSel, "[") {
-			arrayMatches := arrayAccesRegex.FindStringSubmatch(thisSel)
-			if len(arrayMatches) > 0 {
-				// Get the key into the map
-				thisSel = arrayMatches[1]
-
-				// Get the index into the array at the key
-				index, err = strconv.Atoi(arrayMatches[2])
-
-				if err != nil {
-					// This should never happen. If it does, something has gone
-					// seriously wrong. Panic.
-					panic("objx: Array index is not an integer.  Must use array[int].")
-				}
+			if index < len(array) {
+				current = array[index]
+			} else {
+				current = nil
 			}
 		}
-		if curMap, ok := current.(Map); ok {
-			current = map[string]interface{}(curMap)
-		}
-		// get the object in question
-		switch current.(type) {
-		case map[string]interface{}:
-			curMSI := current.(map[string]interface{})
-			if len(selSegs) <= 1 && isSet {
-				curMSI[thisSel] = value
-				return nil
-			}
-			current = curMSI[thisSel]
-		default:
-			current = nil
-		}
-		// do we need to access the item of an array?
-		if index > -1 {
-			if array, ok := current.([]interface{}); ok {
-				if index < len(array) {
-					current = array[index]
-				} else {
-					current = nil
-				}
-			}
-		}
-		if len(selSegs) > 1 {
-			current = access(current, selSegs[1], value, isSet)
-		}
+	}
+	if len(selSegs) > 1 {
+		current = access(current, selSegs[1], value, isSet)
 	}
 	return current
 }
-
-// intFromInterface converts an interface object to the largest
-// representation of an unsigned integer using a type switch and
-// assertions
-func intFromInterface(selector interface{}) int {
-	var value int
-	switch selector.(type) {
-	case int:
-		value = selector.(int)
-	case int8:
-		value = int(selector.(int8))
-	case int16:
-		value = int(selector.(int16))
-	case int32:
-		value = int(selector.(int32))
-	case int64:
-		value = int(selector.(int64))
-	case uint:
-		value = int(selector.(uint))
-	case uint8:
-		value = int(selector.(uint8))
-	case uint16:
-		value = int(selector.(uint16))
-	case uint32:
-		value = int(selector.(uint32))
-	case uint64:
-		value = int(selector.(uint64))
-	default:
-		return 0
-	}
-	return value
-}