diff --git a/.dockerignore b/.dockerignore
index 31445978..3cd880c4 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,7 +5,7 @@
/web/public/
/.vscode/
/.idea/
-/.git/objects/
+/.git/
# source files
docker-compose.yml
diff --git a/.drone.yml b/.drone.yml
index a7cf619a..3c28025a 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,18 +7,22 @@ pipeline:
build:
image: golang:1.10-alpine
commands:
+ - sleep 5
+ - nslookup mongo
+ - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
- cd backend/app
- go build -v ./...
docker_master:
image: plugins/docker
repo: umputun/remark42
- secrets: [ docker_username, docker_password ]
+ secrets: [ docker_username, docker_password, mongo_remark_test ]
build_args:
- DRONE=${DRONE}
- DRONE_TAG=${DRONE_TAG}
- DRONE_COMMIT=${DRONE_COMMIT}
- DRONE_BRANCH=${DRONE_BRANCH}
+ build_args_from_env: [mongo_remark_test]
tags:
- ${DRONE_COMMIT_BRANCH/\//-}
when:
@@ -42,11 +46,12 @@ pipeline:
docker_branch:
image: plugins/docker
repo: umputun/remark42
- secrets: [ docker_username, docker_password ]
+ secrets: [ docker_username, docker_password, mongo_remark_test ]
build_args:
- DRONE=${DRONE}
- DRONE_COMMIT=${DRONE_COMMIT}
- DRONE_BRANCH=${DRONE_BRANCH}
+ build_args_from_env: [mongo_remark_test]
tags:
- ${DRONE_COMMIT_BRANCH/\//-}
when:
@@ -79,3 +84,9 @@ pipeline:
secrets: [ email_username, email_password ]
when:
status: [ changed, failure ]
+
+services:
+ mongo:
+ image: mongo:3.6
+ command: [ --smallfiles ]
+
diff --git a/.gitignore b/.gitignore
index fba2a845..e1385214 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,4 @@ debug.test
*.test
/rest-client.env.json
.DS_Store
+.mongo
diff --git a/.travis.yml b/.travis.yml
index 9cb6efc1..0dd8f7e3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,4 +16,5 @@ script:
--build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
--build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
--build-arg TRAVIS_TAG=$TRAVIS_TAG
+ --build-arg MONGO_REMARK_TEST=$MONGO_REMARK_TEST
.
diff --git a/Dockerfile b/Dockerfile
index 11911d97..b2527561 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,28 +19,39 @@ ARG DRONE_BRANCH
ARG DRONE_PULL_REQUEST
ARG SKIP_BACKEND_TEST
+ARG MONGO_REMARK_TEST
WORKDIR /go/src/github.com/umputun/remark/backend
ADD backend /go/src/github.com/umputun/remark/backend
-RUN cd app && \
+# run tests
+RUN \
+ if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \
+ cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
else echo "skip backend test" ; fi
+RUN echo "mongo=${MONGO_REMARK_TEST}" >> /etc/hosts
+
+# linters
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+ if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \
gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
--enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
--enable=deadcode --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \
else echo "skip backend linters" ; fi
-# coverage test, submit to coverals if COVERALLS_TOKEN in env
-RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
- echo coverall not enabled ; \
- else \
- mkdir -p target && /script/coverage.sh && \
- goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+# coverage report
+RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
+ mkdir -p target && /script/coverage.sh ; \
+ else echo "skip backend coverage" ; fi
-# get revision from git. if DRONE presented use DRONE_* git env to make version
+# submit coverage to coverals if COVERALLS_TOKEN in env
+RUN if [ -z "$COVERALLS_TOKEN" ] ; then \
+ echo "coverall not enabled" ; \
+ else goveralls -coverprofile=.cover/cover.out -service=travis-ci -repotoken $COVERALLS_TOKEN || echo "coverall failed!"; fi
+
+# if DRONE presented use DRONE_* git env to make version
RUN \
if [ -z "$DRONE" ] ; then \
echo "runs outside of drone" && version="local"; \
diff --git a/README.md b/README.md
index ddaa5c59..3a871087 100644
--- a/README.md
+++ b/README.md
@@ -312,7 +312,15 @@ Frontend docker compose config by default skips running backend related tests an
### Backend
-Maybe instructions for local build here?
+In order to run backend locally (development mode, without docker) you have to have latest stable `go` toolchain [installed](https://golang.org/doc/install).
+
+
+To run backend - `go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080`
+It stars backend service with embedded bolt store on port `8080` with basic auth, allowing to authenticate and run requests directly, like this:
+`HTTP http://dev:password@127.0.0.1:8080/api/v1/find?site=remark&sort=-active&format=tree&url=http://127.0.0.1:8080`
+
+To run backend with mongodb store mongo container should be started first - `docker run -d -p 27017:27017 -name=mongo mongo:3.6 --smallfiles` and then
+`go run backend/app/main.go --dbg --secret=12345 --dev-passwd=password --site=remark --url=http://127.0.0.1:8080 --store.type=mongo --store.mongo.url=localhost`
### Frontend
diff --git a/backend/Gopkg.lock b/backend/Gopkg.lock
index 41b0685d..9d919b2a 100644
--- a/backend/Gopkg.lock
+++ b/backend/Gopkg.lock
@@ -54,6 +54,18 @@
packages = ["."]
revision = "6ab5f3083f3d925e1944d58cdaebf43bbbff9238"
+[[projects]]
+ branch = "master"
+ name = "github.com/globalsign/mgo"
+ packages = [
+ ".",
+ "bson",
+ "internal/json",
+ "internal/sasl",
+ "internal/scram"
+ ]
+ revision = "113d3961e7311526535a1ef7042196563d442761"
+
[[projects]]
name = "github.com/go-chi/chi"
packages = [
@@ -238,6 +250,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
- inputs-digest = "38280a8f373ce1799e64ac1709e9748ae8cf6cca30dfeabca807d2db9fe34ab8"
+ inputs-digest = "b6dd6c99dd3a4f5135f8b869063a2b24044494b2ed77442bfd50a45d096e414a"
solver-name = "gps-cdcl"
solver-version = 1
diff --git a/backend/Gopkg.toml b/backend/Gopkg.toml
index 1d2a65e2..03b1bf22 100644
--- a/backend/Gopkg.toml
+++ b/backend/Gopkg.toml
@@ -24,13 +24,21 @@ required = ["github.com/patrickmn/go-cache"]
name = "gopkg.in/russross/blackfriday.v2"
version = "2.0.0"
+[[constraint]]
+ name = "github.com/patrickmn/go-cache"
+ version = "2.1.0"
+
+[[constraint]]
+ branch = "master"
+ name = "github.com/jessevdk/go-flags"
+
+[[constraint]]
+ name = "github.com/globalsign/mgo"
+ version = "r2018.06.15"
+
[prune]
go-tests = true
unused-packages = true
-[[constraint]]
- name = "github.com/patrickmn/go-cache"
- version = "2.1.0"
-[[constraint]]
- branch = "master"
- name = "github.com/jessevdk/go-flags"
+
+
diff --git a/backend/app/main.go b/backend/app/main.go
index b3b31909..9419cf6e 100644
--- a/backend/app/main.go
+++ b/backend/app/main.go
@@ -11,7 +11,10 @@ import (
"syscall"
"time"
+ "github.com/umputun/remark/backend/app/store/avatar"
+
"github.com/coreos/bbolt"
+ "github.com/globalsign/mgo"
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
@@ -22,6 +25,7 @@ import (
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store/engine"
+ "github.com/umputun/remark/backend/app/store/engine/mongo"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -77,6 +81,7 @@ type StoreGroup struct {
Path string `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+ Mongo MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
}
// AvatarGroup defines options group for avatar params
@@ -85,7 +90,8 @@ type AvatarGroup struct {
FS struct {
Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
} `group:"fs" namespace:"fs" env-namespace:"FS"`
- RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+ Mongo MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
+ RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
}
// CacheGroup defines options group for cache params
@@ -98,6 +104,17 @@ type CacheGroup struct {
} `group:"max" namespace:"max" env-namespace:"MAX"`
}
+// MongoOpts holds all mongo params
+type MongoOpts struct {
+ URL string `long:"url" env:"URL" description:"mongo url"`
+ Server []string `long:"server" env:"SERVER" description:"mongo host:port" env-delim:","`
+ DB string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
+ User string `long:"user" env:"USER" default:"" description:"mongo user"`
+ Passwd string `long:"password" env:"PASSWD" default:"" description:"mongo pssword"`
+ SSL bool `long:"ssl" env:"SSL" description:"connect to mongo with ssl"`
+ Dbg bool `long:"dbg" env:"DEBUG" description:"enable mongo debug"`
+}
+
var revision = "unknown"
// Application holds all active objects
@@ -107,6 +124,7 @@ type Application struct {
migratorSrv *api.Migrator
exporter migrator.Exporter
devAuth *auth.DevAuthServer
+ dataService *service.DataStore
terminated chan struct{}
}
@@ -152,13 +170,13 @@ func New(opts Opts) (*Application, error) {
return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
}
- boltStore, err := makeDataStore(opts.Store, opts.Sites)
+ storeEngine, err := makeDataStore(opts.Store, opts.Sites)
if err != nil {
return nil, err
}
dataService := &service.DataStore{
- Interface: boltStore,
+ Interface: storeEngine,
EditDuration: opts.EditDuration,
Secret: opts.SecretKey,
MaxCommentSize: opts.MaxCommentSize,
@@ -232,7 +250,8 @@ func New(opts Opts) (*Application, error) {
}
tch := make(chan struct{})
- return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, Opts: opts, terminated: tch}, nil
+ return &Application{restSrv: srv, migratorSrv: migr, exporter: exporter, devAuth: devAuth, dataService: dataService,
+ Opts: opts, terminated: tch}, nil
}
// Run all application objects
@@ -249,6 +268,10 @@ func (a *Application) Run(ctx context.Context) error {
if a.devAuth != nil {
a.devAuth.Shutdown()
}
+ if e := a.dataService.Close(); e != nil {
+ log.Printf("[WARN] failed to close store, %s", e)
+ }
+
}()
a.activateBackup(ctx) // runs in goroutine for each site
go a.migratorSrv.Run(a.Port + 1) // migrator server runs on +1, localhost only
@@ -284,27 +307,40 @@ func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interfac
switch group.Type {
case "bolt":
if err = makeDirs(group.Bolt.Path); err != nil {
- return nil, err
+ return nil, errors.Wrap(err, "failed to create bolt store")
}
sites := []engine.BoltSite{}
for _, site := range siteNames {
sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", group.Bolt.Path, site)})
}
result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
+ case "mongo":
+ mgServer, e := makeMongo(group.Mongo)
+ if e != nil {
+ return result, errors.Wrap(e, "failed to create mongo server")
+ }
+ conn := mongo.NewConnection(mgServer, group.Mongo.DB, "")
+ result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
default:
return nil, errors.Errorf("unsupported store type %s", group.Type)
}
-
return result, errors.Wrap(err, "can't initialize data store")
}
-func makeAvatarStore(group AvatarGroup) (result proxy.AvatarStore, err error) {
+func makeAvatarStore(group AvatarGroup) (avatar.Store, error) {
switch group.Type {
case "fs":
- if err = makeDirs(group.FS.Path); err != nil {
+ if err := makeDirs(group.FS.Path); err != nil {
return nil, err
}
- return proxy.NewFSAvatarStore(group.FS.Path, group.RszLmt), nil
+ return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil
+ case "mongo":
+ mgServer, err := makeMongo(group.Mongo)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create mongo server")
+ }
+ conn := mongo.NewConnection(mgServer, group.Mongo.DB, "")
+ return avatar.NewGridFS(conn, group.RszLmt), nil
}
return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
}
@@ -338,6 +374,26 @@ func makeDirs(dirs ...string) error {
return nil
}
+func makeMongo(mopts MongoOpts) (result *mongo.Server, err error) {
+ if mopts.URL != "" {
+ log.Print("[DEBUG] mongo url provided")
+ return mongo.NewServerWithURL(mopts.URL, 10*time.Second)
+ }
+ dial := mgo.DialInfo{
+ Addrs: mopts.Server,
+ Database: mopts.DB,
+ Timeout: 10 * time.Second,
+ Username: mopts.User,
+ Password: mopts.Passwd,
+ Source: "admin",
+ }
+
+ return mongo.NewServer(dial, mongo.ServerParams{
+ Debug: mopts.Dbg,
+ SSL: mopts.SSL,
+ })
+}
+
func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {
makeParams := func(cid, secret string) auth.Params {
diff --git a/backend/app/main_test.go b/backend/app/main_test.go
index fb414217..d9e5fc0f 100644
--- a/backend/app/main_test.go
+++ b/backend/app/main_test.go
@@ -72,6 +72,7 @@ func TestApplicationDevMode(t *testing.T) {
app.Wait()
}
+
func TestApplicationFailed(t *testing.T) {
opts := Opts{}
p := flags.NewParser(&opts, flags.Default)
@@ -103,10 +104,12 @@ func TestApplicationFailed(t *testing.T) {
t.Log(err)
opts = Opts{}
- _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=mongo"})
- assert.Nil(t, err)
+ _, err = p.ParseArgs([]string{"--secret=123456", "--url=https://demo.remark42.com", "--backup=/tmp", "--store.type=blah"})
+ assert.NotNil(t, err, "blah is invalid type")
+
+ opts.Store.Type = "blah"
_, err = New(opts)
- assert.EqualError(t, err, "unsupported store type mongo")
+ assert.EqualError(t, err, "unsupported store type blah")
t.Log(err)
}
diff --git a/backend/app/rest/api/migrator.go b/backend/app/rest/api/migrator.go
index 76c3b12f..8e17c9ac 100644
--- a/backend/app/rest/api/migrator.go
+++ b/backend/app/rest/api/migrator.go
@@ -92,7 +92,7 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("provider") == "disqus" {
importer = m.DisqusImporter
}
-
+ log.Printf("[DEBUG] import request for site=%s", siteID)
size, err := importer.Import(r.Body, siteID)
if err != nil {
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
@@ -116,8 +116,9 @@ func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
}
siteID := r.URL.Query().Get("site")
-
exportFile := fmt.Sprintf("%s-%s.json.gz", siteID, time.Now().Format("20060102"))
+ log.Printf("[DEBUG] import request for site=%s to %s", siteID, exportFile)
+
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment;filename="+exportFile)
w.WriteHeader(http.StatusOK)
diff --git a/backend/app/rest/api/rest_test.go b/backend/app/rest/api/rest_test.go
index 7826920d..1813e223 100644
--- a/backend/app/rest/api/rest_test.go
+++ b/backend/app/rest/api/rest_test.go
@@ -19,6 +19,7 @@ import (
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/service"
)
@@ -56,7 +57,7 @@ func TestRest_GetStarted(t *testing.T) {
}
func TestRest_Shutdown(t *testing.T) {
- srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300),
+ srv := Rest{Authenticator: auth.Authenticator{}, AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300),
RoutePath: "/api/v1/avatar"}, ImageProxy: &proxy.Image{}}
go func() {
@@ -92,7 +93,7 @@ func prep(t *testing.T) (srv *Rest, ts *httptest.Server) {
Cache: &mockCache{},
WebRoot: "/tmp",
RemarkURL: "https://demo.remark42.com",
- AvatarProxy: &proxy.Avatar{Store: proxy.NewFSAvatarStore("/tmp", 300), RoutePath: "/api/v1/avatar"},
+ AvatarProxy: &proxy.Avatar{Store: avatar.NewLocalFS("/tmp", 300), RoutePath: "/api/v1/avatar"},
ImageProxy: &proxy.Image{},
ReadOnlyAge: 10,
}
diff --git a/backend/app/rest/proxy/avatar.go b/backend/app/rest/proxy/avatar.go
index 714a5597..4c674ba0 100644
--- a/backend/app/rest/proxy/avatar.go
+++ b/backend/app/rest/proxy/avatar.go
@@ -13,18 +13,17 @@ import (
"github.com/umputun/remark/backend/app/rest"
"github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/avatar"
)
// Avatar provides file-system store and http handler for avatars
// On user login auth will call Put and it will retrieve and save picture locally.
type Avatar struct {
- Store AvatarStore
+ Store avatar.Store
RoutePath string
RemarkURL string
}
-const imgSfx = ".image"
-
// Put stores retrieved avatar to StorePath. Gets image from user info. Returns proxied url
func (p *Avatar) Put(u store.User) (avatarURL string, err error) {
diff --git a/backend/app/rest/proxy/avatar_store.go b/backend/app/rest/proxy/avatar_store.go
deleted file mode 100644
index 713c74c2..00000000
--- a/backend/app/rest/proxy/avatar_store.go
+++ /dev/null
@@ -1,159 +0,0 @@
-package proxy
-
-//go:generate sh -c "mockery -inpkg -name AvatarStore -print > /tmp/mock.tmp && mv /tmp/mock.tmp avatar_store_mock.go"
-
-import (
- "bytes"
- "fmt"
- "hash/crc64"
- "image"
- "image/png"
- "io"
- "log"
- "os"
- "path"
- "strconv"
- "strings"
- "sync"
-
- // Initializing packages for supporting GIF and JPEG formats.
- _ "image/gif"
- _ "image/jpeg"
-
- "github.com/pkg/errors"
- "golang.org/x/image/draw"
-
- "github.com/umputun/remark/backend/app/store"
-)
-
-// AvatarStore defines interface to store and serve avatars
-type AvatarStore interface {
- Put(userID string, reader io.Reader) (avatar string, err error)
- Get(avatar string) (reader io.ReadCloser, size int, err error)
- ID(avatar string) (id string)
-}
-
-// FSAvatarStore implements AvatarStore for local file system
-type FSAvatarStore struct {
- storePath string
- resizeLimit int
- ctcTable *crc64.Table
- once sync.Once
-}
-
-// NewFSAvatarStore makes file-system avatar store
-func NewFSAvatarStore(storePath string, resizeLimit int) *FSAvatarStore {
- return &FSAvatarStore{storePath: storePath, resizeLimit: resizeLimit}
-}
-
-// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
-func (fs *FSAvatarStore) Put(userID string, reader io.Reader) (avatar string, err error) {
- id := store.EncodeID(userID)
- location := fs.location(id) // location adds partition to path
-
- if _, err = os.Stat(location); os.IsNotExist(err) {
- if e := os.Mkdir(location, 0700); e != nil {
- return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
- }
- }
-
- avFile := path.Join(location, id+imgSfx)
- fh, err := os.Create(avFile)
- if err != nil {
- return "", errors.Wrapf(err, "can't create file %s", avFile)
- }
- defer func() {
- if e := fh.Close(); e != nil {
- log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
- }
- }()
-
- // Trying to resize avatar.
- if reader = resize(reader, fs.resizeLimit); reader == nil {
- return "", errors.New("avatar reader is nil")
- }
-
- if _, err = io.Copy(fh, reader); err != nil {
- return "", errors.Wrapf(err, "can't save file %s", avFile)
- }
- return id + imgSfx, nil
-}
-
-// Get avatar reader for avatar id.image
-func (fs *FSAvatarStore) Get(avatar string) (reader io.ReadCloser, size int, err error) {
- location := fs.location(strings.TrimSuffix(avatar, imgSfx))
- avFile := path.Join(location, avatar)
- fh, err := os.Open(avFile)
- if err != nil {
- return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar)
- }
- if fi, e := fh.Stat(); e == nil {
- size = int(fi.Size())
- }
- return fh, size, nil
-}
-
-// ID returns a fingerprint of the avatar content.
-func (fs *FSAvatarStore) ID(avatar string) (id string) {
- location := fs.location(strings.TrimSuffix(avatar, imgSfx))
- avFile := path.Join(location, avatar)
- fi, err := os.Stat(avFile)
- if err != nil {
- log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err)
- return store.EncodeID(avatar)
- }
- return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10))
-}
-
-// get location (directory) for user id by adding partition to final path in order to keep files
-// in different subdirectories and avoid too many files in a single place.
-// the end result is a full path like this - /tmp/avatars.test/92
-func (fs *FSAvatarStore) location(id string) string {
- fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
- checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
- partition := checksum64 % 100
- return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
-}
-
-// Resizes an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
-// (width or height) preserving aspect ratio.
-// Returns original reader if resizing is not needed or failed.
-func resize(reader io.Reader, limit int) io.Reader {
- if reader == nil {
- log.Print("[WARN] avatar resize(): reader is nil")
- return nil
- }
- if limit <= 0 {
- log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
- return reader
- }
-
- var teeBuf bytes.Buffer
- tee := io.TeeReader(reader, &teeBuf)
- src, _, err := image.Decode(tee)
- if err != nil {
- log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
- return &teeBuf
- }
-
- bounds := src.Bounds()
- w, h := bounds.Dx(), bounds.Dy()
- if w <= limit && h <= limit || w <= 0 || h <= 0 {
- log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
- return &teeBuf
- }
- newW, newH := w*limit/h, limit
- if w > h {
- newW, newH = limit, h*limit/w
- }
- m := image.NewRGBA(image.Rect(0, 0, newW, newH))
- // Slower than `draw.ApproxBiLinear.Scale()` but better quality.
- draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
-
- var out bytes.Buffer
- if err = png.Encode(&out, m); err != nil {
- log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
- return &teeBuf
- }
- return &out
-}
diff --git a/backend/app/rest/proxy/avatar_test.go b/backend/app/rest/proxy/avatar_test.go
index db676f13..eb84bb75 100644
--- a/backend/app/rest/proxy/avatar_test.go
+++ b/backend/app/rest/proxy/avatar_test.go
@@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/avatar"
)
func TestAvatar_Put(t *testing.T) {
@@ -30,7 +31,7 @@ func TestAvatar_Put(t *testing.T) {
}))
defer ts.Close()
- p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
+ p := Avatar{RoutePath: "/avatar", RemarkURL: "http://localhost:8080", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
@@ -59,7 +60,7 @@ func TestAvatar_PutFailed(t *testing.T) {
}))
defer ts.Close()
- p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
+ p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
u := store.User{ID: "user1", Name: "user1 name"}
_, err := p.Put(u)
@@ -89,7 +90,7 @@ func TestAvatar_Routes(t *testing.T) {
}))
defer ts.Close()
- p := Avatar{RoutePath: "/avatar", Store: NewFSAvatarStore("/tmp/avatars.test", 300)}
+ p := Avatar{RoutePath: "/avatar", Store: avatar.NewLocalFS("/tmp/avatars.test", 300)}
os.MkdirAll("/tmp/avatars.test", 0700)
defer os.RemoveAll("/tmp/avatars.test")
diff --git a/backend/app/store/avatar/gridfs.go b/backend/app/store/avatar/gridfs.go
new file mode 100644
index 00000000..f18a79b9
--- /dev/null
+++ b/backend/app/store/avatar/gridfs.go
@@ -0,0 +1,83 @@
+package avatar
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "log"
+
+ "github.com/globalsign/mgo"
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/engine/mongo"
+)
+
+// NewGridFS makes gridfs (mongo) avatar store
+func NewGridFS(conn *mongo.Connection, resizeLimit int) *GridFS {
+ return &GridFS{Connection: conn, resizeLimit: resizeLimit}
+}
+
+// GridFS implements Store for GridFS
+type GridFS struct {
+ Connection *mongo.Connection
+ resizeLimit int
+}
+
+// Put avatar to gridfs object, try to resize
+func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
+ id := store.EncodeID(userID)
+ err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
+ fh, e := dbase.GridFS("fs").Create(id + imgSfx)
+ if e != nil {
+ return e
+ }
+ defer func() {
+ if err = fh.Close(); err != nil {
+ log.Printf("[WARN] can't close avatar file %v, %s", fh, err)
+ }
+ }()
+
+ // Trying to resize avatar.
+ if reader = resize(reader, gf.resizeLimit); reader == nil {
+ return errors.New("avatar reader is nil")
+ }
+ _, e = io.Copy(fh, reader)
+ return e
+ })
+ return id + imgSfx, err
+}
+
+// Get avatar reader for avatar id.image
+func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
+ buf := &bytes.Buffer{}
+ err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
+ fh, e := dbase.GridFS("fs").Open(avatar)
+ if e != nil {
+ return errors.Wrapf(e, "can't load avatar %s", avatar)
+ }
+ if _, e = io.Copy(buf, fh); e != nil {
+ return errors.Wrapf(e, "can't copy avatar %s", avatar)
+ }
+ size = int(fh.Size())
+ return fh.Close()
+ })
+ return ioutil.NopCloser(buf), size, err
+}
+
+// ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly
+func (gf *GridFS) ID(avatar string) (id string) {
+ err := gf.Connection.WithDB(func(dbase *mgo.Database) error {
+ fh, e := dbase.GridFS("fs").Open(avatar)
+ if e != nil {
+ return errors.Wrapf(e, "can't open avatar %s", avatar)
+ }
+ id = fh.MD5()
+ return nil
+ })
+ if err != nil {
+ log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err)
+ return store.EncodeID(avatar)
+ }
+ return id
+}
diff --git a/backend/app/store/avatar/gridfs_test.go b/backend/app/store/avatar/gridfs_test.go
new file mode 100644
index 00000000..fe4da914
--- /dev/null
+++ b/backend/app/store/avatar/gridfs_test.go
@@ -0,0 +1,50 @@
+package avatar
+
+import (
+ "io/ioutil"
+ "strings"
+ "testing"
+
+ "github.com/globalsign/mgo"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store/engine/mongo"
+)
+
+func TestGridFS_PutAndGet(t *testing.T) {
+ p, skip := prepGFStore(t)
+ if skip {
+ return
+ }
+ avatar, err := p.Put("user1", strings.NewReader("some picture bin data"))
+ require.Nil(t, err)
+ assert.Equal(t, "b3daa77b4c04a9551b8781d03191fe098f325e67.image", avatar)
+
+ rd, size, err := p.Get(avatar)
+ require.Nil(t, err)
+ assert.Equal(t, 21, size)
+ data, err := ioutil.ReadAll(rd)
+ require.Nil(t, err)
+ assert.Equal(t, "some picture bin data", string(data))
+
+ _, _, err = p.Get("bad avatar")
+ assert.NotNil(t, err)
+
+ assert.Equal(t, "8ce5568f7f9a1c9da5b897bc8642e397", p.ID(avatar))
+ assert.Equal(t, "70c881d4a26984ddce795f6f71817c9cf4480e79", p.ID("aaaa"), "no data, encode avatar id")
+}
+
+func prepGFStore(t *testing.T) (Store, bool) {
+ conn, err := mongo.MakeTestConnection(t)
+ if err != nil {
+ return nil, true
+ }
+ _ = conn.WithCustomCollection("fs.chunks", func(coll *mgo.Collection) error {
+ return coll.DropCollection()
+ })
+ _ = conn.WithCustomCollection("fs.files", func(coll *mgo.Collection) error {
+ return coll.DropCollection()
+ })
+ return NewGridFS(conn, 0), false
+}
diff --git a/backend/app/store/avatar/localfs.go b/backend/app/store/avatar/localfs.go
new file mode 100644
index 00000000..735fa1bf
--- /dev/null
+++ b/backend/app/store/avatar/localfs.go
@@ -0,0 +1,99 @@
+package avatar
+
+import (
+ "fmt"
+ "hash/crc64"
+ "io"
+ "log"
+ "os"
+ "path"
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+)
+
+// LocalFS implements Store for local file system
+type LocalFS struct {
+ storePath string
+ resizeLimit int
+ ctcTable *crc64.Table
+ once sync.Once
+}
+
+// NewLocalFS makes file-system avatar store
+func NewLocalFS(storePath string, resizeLimit int) *LocalFS {
+ return &LocalFS{storePath: storePath, resizeLimit: resizeLimit}
+}
+
+// Put avatar for userID to file and return avatar's file name (base), like 12345678.image
+func (fs *LocalFS) Put(userID string, reader io.Reader) (avatar string, err error) {
+ id := store.EncodeID(userID)
+ location := fs.location(id) // location adds partition to path
+
+ if _, err = os.Stat(location); os.IsNotExist(err) {
+ if e := os.Mkdir(location, 0700); e != nil {
+ return "", errors.Wrapf(e, "failed to mkdir avatar location %s", location)
+ }
+ }
+
+ avFile := path.Join(location, id+imgSfx)
+ fh, err := os.Create(avFile)
+ if err != nil {
+ return "", errors.Wrapf(err, "can't create file %s", avFile)
+ }
+ defer func() {
+ if e := fh.Close(); e != nil {
+ log.Printf("[WARN] can't close avatar file %s, %s", avFile, e)
+ }
+ }()
+
+ // Trying to resize avatar.
+ if reader = resize(reader, fs.resizeLimit); reader == nil {
+ return "", errors.New("avatar reader is nil")
+ }
+
+ if _, err = io.Copy(fh, reader); err != nil {
+ return "", errors.Wrapf(err, "can't save file %s", avFile)
+ }
+ return id + imgSfx, nil
+}
+
+// Get avatar reader for avatar id.image
+func (fs *LocalFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
+ location := fs.location(strings.TrimSuffix(avatar, imgSfx))
+ avFile := path.Join(location, avatar)
+ fh, err := os.Open(avFile)
+ if err != nil {
+ return nil, 0, errors.Wrapf(err, "can't load avatar %s, id", avatar)
+ }
+ if fi, e := fh.Stat(); e == nil {
+ size = int(fi.Size())
+ }
+ return fh, size, nil
+}
+
+// ID returns a fingerprint of the avatar content.
+func (fs *LocalFS) ID(avatar string) (id string) {
+ location := fs.location(strings.TrimSuffix(avatar, imgSfx))
+ avFile := path.Join(location, avatar)
+ fi, err := os.Stat(avFile)
+ if err != nil {
+ log.Printf("[DEBUG] can't get file info '%s', %s", avFile, err)
+ return store.EncodeID(avatar)
+ }
+ return store.EncodeID(avatar + strconv.FormatInt(fi.ModTime().Unix(), 10))
+}
+
+// get location (directory) for user id by adding partition to final path in order to keep files
+// in different subdirectories and avoid too many files in a single place.
+// the end result is a full path like this - /tmp/avatars.test/92
+func (fs *LocalFS) location(id string) string {
+ fs.once.Do(func() { fs.ctcTable = crc64.MakeTable(crc64.ECMA) })
+ checksum64 := crc64.Checksum([]byte(id), fs.ctcTable)
+ partition := checksum64 % 100
+ return path.Join(fs.storePath, fmt.Sprintf("%02d", partition))
+}
diff --git a/backend/app/rest/proxy/avatar_store_test.go b/backend/app/store/avatar/localfs_test.go
similarity index 64%
rename from backend/app/rest/proxy/avatar_store_test.go
rename to backend/app/store/avatar/localfs_test.go
index 34c8291a..f60bd9ed 100644
--- a/backend/app/rest/proxy/avatar_store_test.go
+++ b/backend/app/store/avatar/localfs_test.go
@@ -1,9 +1,6 @@
-package proxy
+package avatar
import (
- "bytes"
- "image"
- "io"
"io/ioutil"
"os"
"strings"
@@ -14,8 +11,8 @@ import (
"github.com/stretchr/testify/require"
)
-func TestAvatarStore_Put(t *testing.T) {
- p := NewFSAvatarStore("/tmp/avatars.test", 300)
+func TestAvatarStoreFS_Put(t *testing.T) {
+ p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
@@ -48,13 +45,13 @@ func TestAvatarStore_Put(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, int64(6986), fi.Size())
- p = NewFSAvatarStore("/dev/null", 300)
+ p = NewLocalFS("/dev/null", 300)
_, err = p.Put("user1", strings.NewReader("some picture bin data"))
assert.EqualError(t, err, "can't create file /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: open /dev/null/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image: not a directory")
}
-func TestAvatarStore_Get(t *testing.T) {
- p := NewFSAvatarStore("/tmp/avatars.test", 300)
+func TestAvatarStoreFS_Get(t *testing.T) {
+ p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
@@ -77,8 +74,8 @@ func TestAvatarStore_Get(t *testing.T) {
assert.Equal(t, "something", string(data))
}
-func TestAvatarStore_Location(t *testing.T) {
- p := NewFSAvatarStore("/tmp/avatars.test", 300)
+func TestAvatarStoreFS_Location(t *testing.T) {
+ p := NewLocalFS("/tmp/avatars.test", 300)
tbl := []struct {
id string
@@ -94,60 +91,8 @@ func TestAvatarStore_Location(t *testing.T) {
}
}
-func TestAvatarStore_resize(t *testing.T) {
- checkC := func(t *testing.T, r io.Reader, cExp []byte) {
- content, err := ioutil.ReadAll(r)
- require.NoError(t, err)
- assert.Equal(t, cExp, content)
- }
-
- // Reader is nil.
- resizedR := resize(nil, 100)
- // assert.EqualError(t, err, "limit should be greater than 0")
- assert.Nil(t, resizedR)
-
- // Negative limit error.
- resizedR = resize(strings.NewReader("some picture bin data"), -1)
- require.NotNil(t, resizedR)
- checkC(t, resizedR, []byte("some picture bin data"))
-
- // Decode error.
- resizedR = resize(strings.NewReader("invalid image content"), 100)
- assert.NotNil(t, resizedR)
- checkC(t, resizedR, []byte("invalid image content"))
-
- cases := []struct {
- file string
- wr, hr int
- }{
- {"testdata/circles.png", 400, 300}, // full size: 800x600 px
- {"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
- }
-
- for _, c := range cases {
- img, err := ioutil.ReadFile(c.file)
- require.Nil(t, err, "can't open test file %s", c.file)
-
- // No need for resize, avatar dimensions are smaller than resize limit.
- resizedR = resize(bytes.NewReader(img), 800)
- assert.NotNilf(t, resizedR, "file %s", c.file)
- checkC(t, resizedR, img)
-
- // Resizing to half of width. Check resizedR avatar format PNG.
- resizedR = resize(bytes.NewReader(img), 400)
- assert.NotNilf(t, resizedR, "file %s", c.file)
-
- imgRz, format, err := image.Decode(resizedR)
- assert.Nilf(t, err, "file %s", c.file)
- assert.Equalf(t, "png", format, "file %s", c.file)
- bounds := imgRz.Bounds()
- assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
- assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
- }
-}
-
-func TestAvatarStore_ID(t *testing.T) {
- p := NewFSAvatarStore("/tmp/avatars.test", 300)
+func TestAvatarStoreFS_ID(t *testing.T) {
+ p := NewLocalFS("/tmp/avatars.test", 300)
err := os.MkdirAll("/tmp/avatars.test/30", 0700)
require.NoError(t, err)
defer os.RemoveAll("/tmp/avatars.test")
@@ -164,8 +109,9 @@ func TestAvatarStore_ID(t *testing.T) {
id = p.ID("b3daa77b4c04a9551b8781d03191fe098f325e67.image")
assert.Equal(t, "325d5b451f32c2f8e7f30a9fd65bff6a42954d9a", id) // store.EncodeID("b3daa77b4c04a9551b8781d03191fe098f325e67.image1500000000")
}
-func BenchmarkAvatarStore_ID(b *testing.B) {
- p := NewFSAvatarStore("/tmp/avatars.test", 300)
+
+func BenchmarkAvatarStoreFS_ID(b *testing.B) {
+ p := NewLocalFS("/tmp/avatars.test", 300)
os.MkdirAll("/tmp/avatars.test/30", 0700)
defer os.RemoveAll("/tmp/avatars.test")
err := ioutil.WriteFile("/tmp/avatars.test/30/b3daa77b4c04a9551b8781d03191fe098f325e67.image", []byte("something"), 0666)
diff --git a/backend/app/store/avatar/store.go b/backend/app/store/avatar/store.go
new file mode 100644
index 00000000..276d3ad6
--- /dev/null
+++ b/backend/app/store/avatar/store.go
@@ -0,0 +1,71 @@
+// Package avatar defines store interface and implements local (fs) and gridfs (mongo) stores.
+package avatar
+
+//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go"
+
+import (
+ "bytes"
+ "image"
+ "image/png"
+ "io"
+ "log"
+
+ // Initializing packages for supporting GIF and JPEG formats.
+ _ "image/gif"
+ _ "image/jpeg"
+
+ "golang.org/x/image/draw"
+)
+
+// imgSfx for avatars
+const imgSfx = ".image"
+
+// Store defines interface to store and and load avatars
+type Store interface {
+ Put(userID string, reader io.Reader) (avatar string, err error)
+ Get(avatar string) (reader io.ReadCloser, size int, err error)
+ ID(avatar string) (id string)
+}
+
+// resize an image of supported format (PNG, JPG, GIF) to the size of "limit" px of the biggest side
+// (width or height) preserving aspect ratio.
+// Returns original reader if resizing is not needed or failed.
+func resize(reader io.Reader, limit int) io.Reader {
+ if reader == nil {
+ log.Print("[WARN] avatar resize(): reader is nil")
+ return nil
+ }
+ if limit <= 0 {
+ log.Print("[DEBUG] avatar resize(): limit should be greater than 0")
+ return reader
+ }
+
+ var teeBuf bytes.Buffer
+ tee := io.TeeReader(reader, &teeBuf)
+ src, _, err := image.Decode(tee)
+ if err != nil {
+ log.Printf("[WARN] avatar resize(): can't decode avatar image, %s", err)
+ return &teeBuf
+ }
+
+ bounds := src.Bounds()
+ w, h := bounds.Dx(), bounds.Dy()
+ if w <= limit && h <= limit || w <= 0 || h <= 0 {
+ log.Print("[DEBUG] resizing image is smaller that the limit or has 0 size")
+ return &teeBuf
+ }
+ newW, newH := w*limit/h, limit
+ if w > h {
+ newW, newH = limit, h*limit/w
+ }
+ m := image.NewRGBA(image.Rect(0, 0, newW, newH))
+ // Slower than `draw.ApproxBiLinear.Scale()` but better quality.
+ draw.BiLinear.Scale(m, m.Bounds(), src, src.Bounds(), draw.Src, nil)
+
+ var out bytes.Buffer
+ if err = png.Encode(&out, m); err != nil {
+ log.Printf("[WARN] avatar resize(): can't encode resized avatar to PNG, %s", err)
+ return &teeBuf
+ }
+ return &out
+}
diff --git a/backend/app/store/avatar/store_test.go b/backend/app/store/avatar/store_test.go
new file mode 100644
index 00000000..a0cac0c9
--- /dev/null
+++ b/backend/app/store/avatar/store_test.go
@@ -0,0 +1,65 @@
+package avatar
+
+import (
+ "bytes"
+ "image"
+ "io"
+ "io/ioutil"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAvatarStore_resize(t *testing.T) {
+ checkC := func(t *testing.T, r io.Reader, cExp []byte) {
+ content, err := ioutil.ReadAll(r)
+ require.NoError(t, err)
+ assert.Equal(t, cExp, content)
+ }
+
+ // Reader is nil.
+ resizedR := resize(nil, 100)
+ // assert.EqualError(t, err, "limit should be greater than 0")
+ assert.Nil(t, resizedR)
+
+ // Negative limit error.
+ resizedR = resize(strings.NewReader("some picture bin data"), -1)
+ require.NotNil(t, resizedR)
+ checkC(t, resizedR, []byte("some picture bin data"))
+
+ // Decode error.
+ resizedR = resize(strings.NewReader("invalid image content"), 100)
+ assert.NotNil(t, resizedR)
+ checkC(t, resizedR, []byte("invalid image content"))
+
+ cases := []struct {
+ file string
+ wr, hr int
+ }{
+ {"testdata/circles.png", 400, 300}, // full size: 800x600 px
+ {"testdata/circles.jpg", 300, 400}, // full size: 600x800 px
+ }
+
+ for _, c := range cases {
+ img, err := ioutil.ReadFile(c.file)
+ require.Nil(t, err, "can't open test file %s", c.file)
+
+ // No need for resize, avatar dimensions are smaller than resize limit.
+ resizedR = resize(bytes.NewReader(img), 800)
+ assert.NotNilf(t, resizedR, "file %s", c.file)
+ checkC(t, resizedR, img)
+
+ // Resizing to half of width. Check resizedR avatar format PNG.
+ resizedR = resize(bytes.NewReader(img), 400)
+ assert.NotNilf(t, resizedR, "file %s", c.file)
+
+ imgRz, format, err := image.Decode(resizedR)
+ assert.Nilf(t, err, "file %s", c.file)
+ assert.Equalf(t, "png", format, "file %s", c.file)
+ bounds := imgRz.Bounds()
+ assert.Equalf(t, c.wr, bounds.Dx(), "file %s", c.file)
+ assert.Equalf(t, c.hr, bounds.Dy(), "file %s", c.file)
+ }
+}
diff --git a/backend/app/rest/proxy/testdata/circles.jpg b/backend/app/store/avatar/testdata/circles.jpg
similarity index 100%
rename from backend/app/rest/proxy/testdata/circles.jpg
rename to backend/app/store/avatar/testdata/circles.jpg
diff --git a/backend/app/rest/proxy/testdata/circles.png b/backend/app/store/avatar/testdata/circles.png
similarity index 100%
rename from backend/app/rest/proxy/testdata/circles.png
rename to backend/app/store/avatar/testdata/circles.png
diff --git a/backend/app/store/comment.go b/backend/app/store/comment.go
index 3d082cc3..0fe916bb 100644
--- a/backend/app/store/comment.go
+++ b/backend/app/store/comment.go
@@ -13,7 +13,7 @@ import (
// Comment represents a single comment with optional reference to its parent
type Comment struct {
- ID string `json:"id"`
+ ID string `json:"id" bson:"_id"`
ParentID string `json:"pid"`
Text string `json:"text"`
Orig string `json:"orig,omitempty"`
@@ -21,21 +21,21 @@ type Comment struct {
Locator Locator `json:"locator"`
Score int `json:"score"`
Votes map[string]bool `json:"votes"`
- Timestamp time.Time `json:"time"`
- Edit *Edit `json:"edit,omitempty"` // pointer to have empty default in json response
- Pin bool `json:"pin,omitempty"`
- Deleted bool `json:"delete,omitempty"`
+ Timestamp time.Time `json:"time" bson:"time"`
+ Edit *Edit `json:"edit,omitempty" bson:"edit,omitempty"` // pointer to have empty default in json response
+ Pin bool `json:"pin,omitempty" bson:"pin,omitempty"`
+ Deleted bool `json:"delete,omitempty" bson:"delete"`
}
// Locator keeps site and url of the post
type Locator struct {
- SiteID string `json:"site,omitempty"`
+ SiteID string `json:"site,omitempty" bson:"site"`
URL string `json:"url"`
}
// Edit indication
type Edit struct {
- Timestamp time.Time `json:"time"`
+ Timestamp time.Time `json:"time" bson:"time"`
Summary string `json:"summary"`
}
@@ -43,9 +43,9 @@ type Edit struct {
type PostInfo struct {
URL string `json:"url"`
Count int `json:"count"`
- ReadOnly bool `json:"read_only,omitempty"`
- FirstTS time.Time `json:"first_time,omitempty"`
- LastTS time.Time `json:"last_time,omitempty"`
+ ReadOnly bool `json:"read_only,omitempty" bson:"read_only,omitempty"`
+ FirstTS time.Time `json:"first_time,omitempty" bson:"first_time,omitempty"`
+ LastTS time.Time `json:"last_time,omitempty" bson:"last_time,omitempty"`
}
// BlockedUser holds id and ts for blocked user
diff --git a/backend/app/store/engine/bolt_accessor.go b/backend/app/store/engine/bolt_accessor.go
index 59bee12f..9ee88c8e 100644
--- a/backend/app/store/engine/bolt_accessor.go
+++ b/backend/app/store/engine/bolt_accessor.go
@@ -7,6 +7,8 @@ import (
"strings"
"time"
+ "github.com/hashicorp/go-multierror"
+
"github.com/coreos/bbolt"
"github.com/pkg/errors"
@@ -37,13 +39,9 @@ const (
readonlyBucketName = "readonly"
verifiedBucketName = "verified"
- // limits
- lastLimit = 1000
- userLimit = 500
+ tsNano = "2006-01-02T15:04:05.000000000Z07:00"
)
-const tsNano = "2006-01-02T15:04:05.000000000Z07:00"
-
// BoltSite defines single site param
type BoltSite struct {
FileName string // full path to boltdb
@@ -411,6 +409,16 @@ func (b *BoltDB) Put(locator store.Locator, comment store.Comment) error {
})
}
+// Close boltdb store
+func (b *BoltDB) Close() error {
+ errs := new(multierror.Error)
+ for site, db := range b.dbs {
+ err := errors.Wrapf(db.Close(), "can't close site %s", site)
+ errs = multierror.Append(errs, err)
+ }
+ return errs.ErrorOrNil()
+}
+
// getPostBucket return bucket with all comments for postURL
func (b *BoltDB) getPostBucket(tx *bolt.Tx, postURL string) (*bolt.Bucket, error) {
postsBkt := tx.Bucket([]byte(postsBucketName))
diff --git a/backend/app/store/engine/bolt_accessor_test.go b/backend/app/store/engine/bolt_accessor_test.go
index 9e340c28..8d2ba847 100644
--- a/backend/app/store/engine/bolt_accessor_test.go
+++ b/backend/app/store/engine/bolt_accessor_test.go
@@ -32,6 +32,8 @@ func TestBoltDB_CreateAndFind(t *testing.T) {
_, err = b.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-bad"}, "time")
assert.EqualError(t, err, `site "radio-t-bad" not found`)
+
+ assert.NoError(t, b.Close())
}
func TestBoltDB_CreateReadOnly(t *testing.T) {
diff --git a/backend/app/store/engine/bolt_admin.go b/backend/app/store/engine/bolt_admin.go
index 0dcd04c5..3d4b6cf5 100644
--- a/backend/app/store/engine/bolt_admin.go
+++ b/backend/app/store/engine/bolt_admin.go
@@ -266,7 +266,7 @@ func (b *BoltDB) SetReadOnly(locator store.Locator, status bool) error {
})
}
-// IsReadOnly checks if user blocked
+// IsReadOnly checks if post in RO mode
func (b *BoltDB) IsReadOnly(locator store.Locator) (ro bool) {
bdb, err := b.db(locator.SiteID)
diff --git a/backend/app/store/engine/engine.go b/backend/app/store/engine/engine.go
index 3b75c115..507efa8c 100644
--- a/backend/app/store/engine/engine.go
+++ b/backend/app/store/engine/engine.go
@@ -3,6 +3,7 @@
package engine
import (
+ "io"
"sort"
"strings"
"time"
@@ -38,6 +39,7 @@ type Accessor interface {
Count(locator store.Locator) (int, error) // number of comments for the post
List(siteID string, limit int, skip int) ([]store.PostInfo, error) // list of commented posts
Info(locator store.Locator, readonlyAge int) (store.PostInfo, error) // get post info
+ Close() error // close/stop engine
}
// Admin defines all store ops avail for admin only
@@ -54,6 +56,19 @@ type Admin interface {
IsVerified(siteID string, userID string) bool // check verified status
}
+// Avatar defines interface for avatar store
+type Avatar interface {
+ Put(userID string, reader io.Reader) (avatar string, err error)
+ Get(avatar string) (reader io.ReadCloser, size int, err error)
+ ID(avatar string) (id string)
+}
+
+const (
+ // limits
+ lastLimit = 1000
+ userLimit = 500
+)
+
// sortComments is for engines can't sort data internally
func sortComments(comments []store.Comment, sortFld string) []store.Comment {
sort.Slice(comments, func(i, j int) bool {
diff --git a/backend/app/store/engine/mongo.go b/backend/app/store/engine/mongo.go
new file mode 100644
index 00000000..8cde07ae
--- /dev/null
+++ b/backend/app/store/engine/mongo.go
@@ -0,0 +1,362 @@
+package engine
+
+import (
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ "github.com/hashicorp/go-multierror"
+ "github.com/pkg/errors"
+
+ "github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/engine/mongo"
+)
+
+// Mongo implements engine interface
+type Mongo struct {
+ conn *mongo.Connection
+ postWriter mongo.BufferedWriter
+}
+
+const (
+ mongoPosts = "posts"
+ mongoMetaPosts = "meta_posts"
+ mongoMetaUsers = "meta_users"
+)
+
+type metaPost struct {
+ ID string `bson:"_id"` // url
+ SiteID string `bson:"site"`
+ ReadOnly bool `bson:"read_only"`
+}
+
+type metaUser struct {
+ ID string `bson:"_id"` // user_id
+ SiteID string `bson:"site"`
+ Verified bool `bson:"verified"`
+ Blocked bool `bson:"blocked"`
+ BlockedUntil time.Time `bson:"blocked_until"`
+}
+
+// NewMongo makes mongo engine. bufferSize denies how many records will be buffered, 0 turns buffering off.
+// flushDuration triggers automatic flus (write from buffer), 0 disables it and will flush as buffer size reached.
+// important! don't use flushDuration=0 for production use as it can leave records in-fly state for long or even unlimited time.
+func NewMongo(conn *mongo.Connection, bufferSize int, flushDuration time.Duration) (*Mongo, error) {
+ writer := mongo.NewBufferedWriter(bufferSize, conn).WithCollection(mongoPosts).WithAutoFlush(flushDuration)
+ result := Mongo{conn: conn, postWriter: writer}
+ err := result.prepare()
+ return &result, errors.Wrap(err, "failed to prepare mongo")
+}
+
+// Create new comment, write can be buffered and delayed.
+func (m *Mongo) Create(comment store.Comment) (commentID string, err error) {
+ // err = m.postWriter.Write(comment)
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ return coll.Insert(&comment)
+ })
+ return comment.ID, err
+}
+
+// Find returns all comments for post and sorts results
+func (m *Mongo) Find(locator store.Locator, sortFld string) (comments []store.Comment, err error) {
+ comments = []store.Comment{}
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}
+ return coll.Find(query).Sort(sortFld).All(&comments)
+ })
+ return comments, err
+}
+
+// Get returns comment for locator.URL and commentID string
+func (m *Mongo) Get(locator store.Locator, commentID string) (comment store.Comment, err error) {
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ query := bson.M{"_id": commentID, "locator.site": locator.SiteID, "locator.url": locator.URL}
+ return coll.Find(query).One(&comment)
+ })
+ return comment, err
+}
+
+// Put updates comment for locator.URL with mutable part of comment
+func (m *Mongo) Put(locator store.Locator, comment store.Comment) error {
+ return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ return coll.Update(bson.M{"_id": comment.ID, "locator.site": locator.SiteID, "locator.url": locator.URL},
+ bson.M{"$set": bson.M{
+ "text": comment.Text,
+ "orig": comment.Orig,
+ "score": comment.Score,
+ "votes": comment.Votes,
+ "pin": comment.Pin,
+ "deleted": comment.Deleted,
+ }})
+ })
+}
+
+// Last returns up to max last comments for given siteID
+func (m *Mongo) Last(siteID string, max int) (comments []store.Comment, err error) {
+ comments = []store.Comment{}
+ if max > lastLimit || max == 0 {
+ max = lastLimit
+ }
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ query := bson.M{"locator.site": siteID, "delete": false}
+ return coll.Find(query).Sort("-time").Limit(max).All(&comments)
+ })
+ return comments, err
+}
+
+// Count returns number of comments for locator
+func (m *Mongo) Count(locator store.Locator) (count int, err error) {
+
+ e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ query := bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "delete": false}
+ count, err = coll.Find(query).Count()
+ return err
+ })
+ return count, e
+}
+
+// List returns list of all commented posts with counters
+func (m *Mongo) List(siteID string, limit, skip int) (list []store.PostInfo, err error) {
+ list = []store.PostInfo{}
+
+ if limit <= 0 {
+ limit = 1000
+ }
+ if skip < 0 {
+ skip = 0
+ }
+
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ pipeline := coll.Pipe([]bson.M{
+ {"$match": bson.M{"locator.site": siteID}},
+ {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
+ {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
+ "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
+ {"$skip": skip},
+ {"$limit": limit},
+ })
+ return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
+ })
+ return list, errors.Wrap(err, "can't get list")
+}
+
+// Info returns time range and count for locator
+func (m *Mongo) Info(locator store.Locator, readOnlyAge int) (info store.PostInfo, err error) {
+ list := []store.PostInfo{}
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ pipeline := coll.Pipe([]bson.M{
+ {"$match": bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL}},
+ {"$project": bson.M{"locator.site": 1, "locator.url": 1, "time": 1}},
+ {"$group": bson.M{"_id": "$locator.url", "url": bson.M{"$first": "$locator.url"}, "count": bson.M{"$sum": 1},
+ "first_time": bson.M{"$min": "$time"}, "last_time": bson.M{"$max": "$time"}}},
+ })
+ return errors.Wrap(pipeline.AllowDiskUse().All(&list), "list pipeline failed")
+ })
+ if err != nil {
+ return info, err
+ }
+ if len(list) == 0 {
+ return info, errors.Errorf("can't load info for %s", locator.URL)
+ }
+ info = list[0]
+ // set read-only from age and manual bucket
+ info.ReadOnly = readOnlyAge > 0 && !info.FirstTS.IsZero() && info.FirstTS.AddDate(0, 0, readOnlyAge).Before(time.Now())
+ if m.IsReadOnly(locator) {
+ info.ReadOnly = true
+ }
+ return info, nil
+}
+
+// User extracts all comments for given site and given userID
+func (m *Mongo) User(siteID, userID string, limit, skip int) (comments []store.Comment, err error) {
+ comments = []store.Comment{}
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ query := bson.M{"locator.site": siteID, "user.id": userID}
+ return m.setLimitAndSkip(coll.Find(query).Sort("-time"), limit, skip).All(&comments)
+ })
+ return comments, errors.Wrapf(err, "can't get comments for user %s", userID)
+}
+
+// UserCount returns number of comments for user
+func (m *Mongo) UserCount(siteID, userID string) (count int, err error) {
+ err = m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ var e error
+ count, e = coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).Count()
+ return e
+ })
+ return count, errors.Wrapf(err, "can't get comments count for user %s", userID)
+}
+
+// SetReadOnly makes post read-only or reset the ro flag
+func (m *Mongo) SetReadOnly(locator store.Locator, status bool) (err error) {
+ return m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
+ _, e := coll.Upsert(bson.M{"_id": locator.URL, "site": locator.SiteID}, bson.M{"$set": bson.M{"read_only": status}})
+ return e
+ })
+}
+
+// IsReadOnly checks if post in RO
+func (m *Mongo) IsReadOnly(locator store.Locator) (ro bool) {
+ meta := metaPost{}
+ err := m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"_id": locator.URL, "site": locator.SiteID}).One(&meta)
+ })
+ return err == nil && meta.ReadOnly
+}
+
+// SetVerified makes user verified or reset the flag
+func (m *Mongo) SetVerified(siteID string, userID string, status bool) error {
+ return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID}, bson.M{"$set": bson.M{"verified": status}})
+ return e
+ })
+}
+
+// IsVerified checks if user verified
+func (m *Mongo) IsVerified(siteID string, userID string) (verified bool) {
+ meta := metaUser{}
+ err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
+ })
+ return err == nil && meta.Verified
+}
+
+// SetBlock blocks/unblocks user for given site. ttl defines for for how long, 0 - permanent
+// block uses blocksBucketName with key=userID and val=TTL+now
+func (m *Mongo) SetBlock(siteID string, userID string, status bool, ttl time.Duration) error {
+
+ until := time.Time{}
+ if status {
+ until = time.Now().AddDate(100, 0, 0) // permanent is 50year
+ if ttl > 0 {
+ until = time.Now().Add(ttl)
+ }
+ }
+ return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ _, e := coll.Upsert(bson.M{"_id": userID, "site": siteID},
+ bson.M{"$set": bson.M{"blocked": status, "blocked_until": until}})
+ return errors.Wrapf(e, "failed to set block for %s", userID)
+ })
+}
+
+// IsBlocked checks if user blocked
+func (m *Mongo) IsBlocked(siteID string, userID string) (blocked bool) {
+ meta := metaUser{}
+ err := m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"_id": userID, "site": siteID}).One(&meta)
+ })
+ return err == nil && meta.Blocked && meta.BlockedUntil.After(time.Now())
+}
+
+// Blocked get lists of blocked users for given site
+func (m *Mongo) Blocked(siteID string) (users []store.BlockedUser, err error) {
+ users = []store.BlockedUser{}
+ metas := []metaUser{}
+ err = m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"site": siteID,
+ "blocked": true, "blocked_until": bson.M{"$gt": time.Now()}}).All(&metas)
+ })
+ if err != nil {
+ return users, errors.Wrapf(err, "can't get blocked users for site for %s", siteID)
+ }
+
+ for _, mu := range metas {
+ blockedUser := store.BlockedUser{ID: mu.ID, Until: mu.BlockedUntil}
+ if ucc, e := m.User(siteID, mu.ID, 1, 0); e == nil && len(ucc) > 0 {
+ blockedUser.Name = ucc[0].User.Name
+ }
+ users = append(users, blockedUser)
+ }
+ return users, nil
+}
+
+// Delete removes comment, by locator from the store.
+// Posts collection only sets status to deleted and clear fields in order to prevent breaking trees of replies.
+func (m *Mongo) Delete(locator store.Locator, commentID string, mode store.DeleteMode) error {
+ comment := store.Comment{}
+ err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ e := coll.Find(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}).One(&comment)
+ if e != nil {
+ return e
+ }
+ comment.SetDeleted(mode)
+ return coll.Update(bson.M{"locator.site": locator.SiteID, "locator.url": locator.URL, "_id": commentID}, comment)
+ })
+ return errors.Wrapf(err, "can't delete %s", commentID)
+}
+
+// DeleteAll removes all info about siteID
+func (m *Mongo) DeleteAll(siteID string) error {
+ err := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ _, e := coll.RemoveAll(bson.M{"locator.site": siteID})
+ return e
+ })
+ return errors.Wrapf(err, "can't delete site %s", siteID)
+}
+
+// DeleteUser removes all comments for given user. Everything will be market as deleted
+// and user name and userID will be changed to "deleted".
+func (m *Mongo) DeleteUser(siteID string, userID string) error {
+ comments := []store.Comment{}
+ return m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ e := coll.Find(bson.M{"locator.site": siteID, "user.id": userID}).All(&comments)
+ if e != nil {
+ return e
+ }
+ for _, c := range comments {
+ if e = m.Delete(c.Locator, c.ID, store.HardDelete); e != nil {
+ return e
+ }
+ }
+ return nil
+ })
+}
+
+// Close boltdb store
+func (m *Mongo) Close() error {
+ if m.postWriter != nil {
+ return m.postWriter.Close()
+ }
+ return nil
+}
+
+// prepare collections with all indexes
+func (m *Mongo) prepare() error {
+ errs := new(multierror.Error)
+ e := m.conn.WithCustomCollection(mongoPosts, func(coll *mgo.Collection) error {
+ errs = multierror.Append(errs, coll.EnsureIndexKey("user.id", "locator.site", "time"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "time"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("locator.site", "time"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("locator.url", "locator.site", "score"))
+ return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoPosts)
+ })
+ if e != nil {
+ return e
+ }
+
+ e = m.conn.WithCustomCollection(mongoMetaPosts, func(coll *mgo.Collection) error {
+ errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("site", "read_only"))
+ return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaPosts)
+ })
+ if e != nil {
+ return e
+ }
+
+ return m.conn.WithCustomCollection(mongoMetaUsers, func(coll *mgo.Collection) error {
+ errs = multierror.Append(errs, coll.EnsureIndexKey("_id", "site"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("site", "blocked"))
+ errs = multierror.Append(errs, coll.EnsureIndexKey("site", "verified"))
+ return errors.Wrapf(errs.ErrorOrNil(), "can't create index for %s", mongoMetaUsers)
+ })
+}
+
+func (m *Mongo) setLimitAndSkip(q *mgo.Query, limit, skip int) *mgo.Query {
+ if limit <= 0 {
+ limit = 1000
+ }
+ if skip < 0 {
+ skip = 0
+ }
+ return q.Skip(skip).Limit(limit)
+}
diff --git a/backend/app/store/engine/mongo/connection.go b/backend/app/store/engine/mongo/connection.go
new file mode 100644
index 00000000..769c8dbc
--- /dev/null
+++ b/backend/app/store/engine/mongo/connection.go
@@ -0,0 +1,67 @@
+package mongo
+
+import (
+ "fmt"
+
+ "github.com/globalsign/mgo"
+)
+
+// sessionFn is a function for all With calls, terminator not supported
+type sessionFn func(coll *mgo.Collection) error
+
+// Connection allows to run request in separate session, closing automatically
+type Connection struct {
+ server *Server
+ db, collection string
+}
+
+// NewConnection makes a connection for server
+func NewConnection(server *Server, db string, collection string) *Connection {
+ return &Connection{server: server, db: db, collection: collection}
+}
+
+// WithCollection passes fun with mgo.Collection from session copy, closes it after done,
+// uses Connection.DB and Connection.Collection
+func (c *Connection) WithCollection(fun sessionFn) (err error) {
+ return c.WithCustomCollection(c.collection, fun)
+}
+
+// WithCustomCollection passes fun with mgo.Collection from session copy, closes it after done
+// uses Connection.DB or (if not defined) dial.Database, and user-defined collection
+func (c *Connection) WithCustomCollection(collection string, fun sessionFn) (err error) {
+ db := c.server.dial.Database
+ if c.db != "" {
+ db = c.db
+ }
+ return c.WithCustomDbCollection(db, collection, fun)
+}
+
+// WithCustomDbCollection passed fun with mgo.Collection from session copy, closes it after done
+// uses passed db and collection directly.
+func (c *Connection) WithCustomDbCollection(db string, collection string, fun sessionFn) (err error) {
+ session := c.server.SessionCopy()
+ defer session.Close()
+ return fun(session.DB(db).C(collection))
+}
+
+// WithDB passes fun with mgo.Database from session copy, closes it after done
+// uses Connection.DB or (if not defined) dial.Database
+func (c *Connection) WithDB(fun func(dbase *mgo.Database) error) (err error) {
+ db := c.server.dial.Database
+ if c.db != "" {
+ db = c.db
+ }
+ return c.WithCustomDB(db, fun)
+}
+
+// WithCustomDB passes fun with mgo.Database from session copy, closes it after done
+// uses passed db directly
+func (c *Connection) WithCustomDB(db string, fun func(dbase *mgo.Database) error) (err error) {
+ session := c.server.SessionCopy()
+ defer session.Close()
+ return fun(session.DB(db))
+}
+
+func (c *Connection) String() string {
+ return fmt.Sprintf("mongo:%s, db:%s, collection:%s", c.server, c.db, c.collection)
+}
diff --git a/backend/app/store/engine/mongo/connection_test.go b/backend/app/store/engine/mongo/connection_test.go
new file mode 100644
index 00000000..0d611647
--- /dev/null
+++ b/backend/app/store/engine/mongo/connection_test.go
@@ -0,0 +1,126 @@
+package mongo
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ "github.com/hashicorp/go-multierror"
+ "github.com/stretchr/testify/assert"
+)
+
+type testRecord struct {
+ Symbol string
+ Num int
+}
+
+func TestConnection_WithCollection(t *testing.T) {
+ c, err := write(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, c)
+
+ var res []testRecord
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 100, len(res))
+
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"symbol": "blah"}).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+
+ r1 := testRecord{}
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"symbol": "blah"}).One(&r1)
+ })
+ assert.Equal(t, mgo.ErrNotFound, err)
+
+ c = NewConnection(c.server, "test", "bbbbbbbaaad")
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Find(bson.M{"symbol": "blah"}).One(&r1)
+ })
+ assert.Equal(t, mgo.ErrNotFound, err)
+}
+
+func TestConnection_WithCollectionNoDB(t *testing.T) {
+ c, err := write(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, c)
+
+ var res []testRecord
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 100, len(res))
+}
+
+func TestConnection_WithDB(t *testing.T) {
+ c, err := write(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, c)
+
+ var res []testRecord
+ err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
+ return dbase.C(c.collection).Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 100, len(res))
+
+ err = c.WithDB(func(dbase *mgo.Database) error {
+ return dbase.C(c.collection).Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 100, len(res))
+}
+
+func TestCleanup(t *testing.T) {
+ c, err := write(t)
+ if err != nil {
+ return
+ }
+ var res []testRecord
+ err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
+ return dbase.C(c.collection).Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 100, len(res))
+
+ RemoveTestCollections(t, c, c.collection)
+ err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
+ return dbase.C(c.collection).Find(nil).All(&res)
+ })
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func write(t *testing.T) (*Connection, error) {
+ c, err := MakeTestConnection(t)
+ if err != nil {
+ return nil, err
+ }
+ err = c.WithCollection(func(coll *mgo.Collection) error {
+ errs := new(multierror.Error)
+ for i := 0; i < 100; i++ {
+ r := testRecord{
+ Symbol: fmt.Sprintf("symb-%02d", i%5),
+ Num: i,
+ }
+ insertErr := coll.Insert(r)
+ assert.Nil(t, insertErr, fmt.Sprintf("insert %+v", r))
+ errs = multierror.Append(errs, insertErr)
+ }
+ return errs.ErrorOrNil()
+ })
+ return c, err
+}
diff --git a/backend/app/store/engine/mongo/server.go b/backend/app/store/engine/mongo/server.go
new file mode 100644
index 00000000..e3a95bb5
--- /dev/null
+++ b/backend/app/store/engine/mongo/server.go
@@ -0,0 +1,121 @@
+// Package mongo wraps mgo to provide easier way to construct mongo server (with auth).
+// Connection provides With* func warapers to run query with session copy
+package mongo
+
+import (
+ "crypto/tls"
+ "fmt"
+ "log"
+ "net"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/pkg/errors"
+)
+
+// Server represents mongo instance and provides session accessor
+type Server struct {
+ dial mgo.DialInfo
+ params ServerParams
+ sess *mgo.Session
+}
+
+// ServerParams optional set of parameters
+type ServerParams struct {
+ Delay int // initial delay to give mongo server some time to start, in case if mongo part of the same compose
+ Debug bool // turn on mgo debug mode
+ ConsistencyMode mgo.Mode
+ SSL bool
+}
+
+// NewServerWithURL makes mongo server from url like
+// mongodb://remark42:password@127.0.0.1t:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin
+func NewServerWithURL(url string, timeout time.Duration) (res *Server, err error) {
+ dial, params, err := parseURL(url, timeout)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create mongo server from url %s", url)
+ }
+ return NewServer(dial, params)
+}
+
+// NewServer doing auth if passwd != "" and can delay to make sure local mongo is up
+func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) {
+ log.Printf("[INFO] make new mongo server %v with %+v", dial.Addrs, params)
+ result := Server{dial: dial, params: params}
+
+ if params.Debug {
+ mgo.SetDebug(true)
+ mgo.SetLogger(log.New(os.Stdout, "MGO ", log.Ldate|log.Ltime|log.Lmicroseconds))
+ }
+
+ if len(dial.Addrs) == 0 {
+ return nil, errors.New("missing mongo address")
+ }
+
+ if params.Delay > 0 {
+ log.Printf("[DEBUG] initial mongo delay=%d", params.Delay)
+ time.Sleep(time.Duration(params.Delay) * time.Second)
+ }
+
+ log.Printf("[DEBUG] dial mongo %s, ssl=%v", dial.Addrs, params.SSL)
+
+ if params.SSL {
+ tlsConfig := &tls.Config{}
+ dial.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
+ conn, e := tls.Dial("tcp", addr.String(), tlsConfig)
+ return conn, e
+ }
+ }
+
+ session, err := mgo.DialWithInfo(&dial)
+ if err != nil {
+ err = fmt.Errorf("can't connect to mongo, %v", err)
+ log.Printf("[ERROR] %v", err)
+ return nil, err
+ }
+ session.SetMode(params.ConsistencyMode, true)
+ session.SetSyncTimeout(30 * time.Second)
+ session.SetSocketTimeout(dial.Timeout)
+
+ if dial.Username != "" && dial.Password != "" {
+ creds := &mgo.Credential{Username: dial.Username, Password: dial.Password, Source: dial.Source}
+ log.Printf("[DEBUG] login to mongo, user=%s, db=%s", creds.Username, creds.Source)
+ if err = session.Login(creds); err != nil {
+ log.Printf("[ERROR] can't login to mongo, %v", err)
+ return nil, err
+ }
+ }
+
+ result.sess = session
+ return &result, nil
+}
+
+// SessionCopy returns copy of main session. Client should close it
+func (m Server) SessionCopy() *mgo.Session {
+ return m.sess.Copy()
+}
+
+func (m Server) String() string {
+ return fmt.Sprintf("%v%s", m.dial.Addrs, m.dial.Database)
+}
+
+// parseURL extends mgo with debug option and extracts ssl flag to make ServerParams
+func parseURL(mongoURL string, connectTimeout time.Duration) (mgo.DialInfo, ServerParams, error) {
+ params := ServerParams{
+ ConsistencyMode: mgo.Monotonic,
+ SSL: strings.Contains(mongoURL, "ssl=true"),
+ Debug: strings.Contains(mongoURL, "debug=true"),
+ }
+
+ mongoURL = strings.Replace(mongoURL, "&debug=true", "", 1)
+ mongoURL = strings.Replace(mongoURL, "?debug=true", "", 1)
+
+ dial, err := mgo.ParseURL(mongoURL)
+ if err != nil {
+ return mgo.DialInfo{}, ServerParams{}, errors.Wrapf(err, "failed to pars mongo url %s", mongoURL)
+ }
+ dial.Timeout = connectTimeout
+ return *dial, params, nil
+}
diff --git a/backend/app/store/engine/mongo/server_test.go b/backend/app/store/engine/mongo/server_test.go
new file mode 100644
index 00000000..466db5d6
--- /dev/null
+++ b/backend/app/store/engine/mongo/server_test.go
@@ -0,0 +1,96 @@
+package mongo
+
+import (
+ "log"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestServer_NewServerGood(t *testing.T) {
+ mongoURL := os.Getenv("MONGO_REMARK_TEST")
+ if mongoURL == "" {
+ mongoURL = "mongodb://mongo:27017/test?debug=true"
+ log.Printf("[WARN] no MONGO_REMARK_TEST in env")
+ }
+ if mongoURL == "skip" {
+ return
+ }
+ m, err := NewServerWithURL(mongoURL, 3*time.Second)
+ assert.Nil(t, err)
+ assert.NotNil(t, m)
+ assert.True(t, m.String() != "")
+}
+
+func TestServer_NewServerBad(t *testing.T) {
+ _, err := NewServerWithURL("mongodb://127.0.0.3:27017/test", 100*time.Millisecond)
+ assert.NotNil(t, err)
+ t.Log(err)
+
+ _, err = NewServer(mgo.DialInfo{Addrs: []string{"127.0.0.2"}, Timeout: 100 * time.Millisecond}, ServerParams{})
+ assert.NotNil(t, err)
+
+ _, err = NewServer(mgo.DialInfo{}, ServerParams{})
+ assert.NotNil(t, err)
+
+ _, err = NewServerWithURL("mongodb://mongo:27017/test?blah=xxx", 100*time.Millisecond)
+ assert.NotNil(t, err)
+ t.Log(err)
+}
+
+func TestServer_parse(t *testing.T) {
+ tbl := []struct {
+ mongoURL string
+ timeout time.Duration
+ params ServerParams
+ dial mgo.DialInfo
+ isErr bool
+ }{
+ {
+ "mongodb://127.0.0.3:27017/test", time.Millisecond,
+ ServerParams{ConsistencyMode: 1},
+ mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test",
+ ReadPreference: &mgo.ReadPreference{Mode: 2}},
+ false,
+ },
+ {
+ "mongodb://user:passwd@127.0.0.3:27017/test?ssl=true&authSource=admin", time.Millisecond,
+ ServerParams{ConsistencyMode: 1, SSL: true},
+ mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test", Source: "admin",
+ Username: "user", Password: "passwd", ReadPreference: &mgo.ReadPreference{Mode: 2}},
+ false,
+ },
+ {
+ "mongodb://127.0.0.3", time.Millisecond,
+ ServerParams{ConsistencyMode: 1, SSL: false},
+ mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}},
+ false,
+ },
+ {
+ "127.0.0.3", time.Millisecond,
+ ServerParams{ConsistencyMode: 1, SSL: false},
+ mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}},
+ false,
+ },
+ {
+ "127.0.0.3?xxx=yyy", time.Millisecond,
+ ServerParams{}, mgo.DialInfo{},
+ true,
+ },
+ }
+
+ for i, tt := range tbl {
+ dial, params, err := parseURL(tt.mongoURL, tt.timeout)
+ dial.DialServer = nil
+ if tt.isErr {
+ assert.NotNil(t, err, "expect error #%d", i)
+ t.Logf("dial %+v, params %+v", dial, params)
+ continue
+ }
+ assert.Equal(t, tt.dial, dial, "test #%d", i)
+ assert.Equal(t, tt.params, params, "test #%d", i)
+ }
+}
diff --git a/backend/app/store/engine/mongo/testing.go b/backend/app/store/engine/mongo/testing.go
new file mode 100644
index 00000000..d013f850
--- /dev/null
+++ b/backend/app/store/engine/mongo/testing.go
@@ -0,0 +1,73 @@
+package mongo
+
+import (
+ "fmt"
+ "log"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+var conn *Connection
+var once sync.Once
+
+// MakeTestConnection connects to MONGO_REMARK_TEST url or "mongo" host (in no env) and returns new connection.
+// collection name randomized on each call
+func MakeTestConnection(t *testing.T) (*Connection, error) {
+ mongoURL := os.Getenv("MONGO_REMARK_TEST")
+ if mongoURL == "" {
+ mongoURL = "mongodb://mongo:27017"
+ log.Printf("[WARN] no MONGO_REMARK_TEST in env")
+ }
+ if mongoURL == "skip" {
+ log.Print("skip mongo test")
+ return nil, errors.New("skip")
+ }
+
+ once.Do(func() {
+ log.Print("[DEBUG] connect to mongo test instance")
+ srv, err := NewServerWithURL(mongoURL, 10*time.Second)
+ assert.Nil(t, err, "failed to dial")
+ collName := fmt.Sprintf("remark42_test_%d", time.Now().Nanosecond())
+ conn = NewConnection(srv, "test", collName)
+ })
+ RemoveTestCollection(t, conn)
+ return conn, nil
+}
+
+// RemoveTestCollection removes all records and drop collection from connection
+func RemoveTestCollection(t *testing.T, c *Connection) {
+ log.Printf("[DEBUG] clean test collection %+v", c.collection)
+ _ = c.WithCollection(func(coll *mgo.Collection) error {
+ _, e := coll.RemoveAll(nil)
+ require.Nil(t, e, "failed to remove records, %s", e)
+ e = coll.DropCollection()
+ if e != nil && e.Error() != "ns not found" {
+ require.Nil(t, e, "failed to drop collection, %s", e)
+ }
+ return e
+ })
+}
+
+// RemoveTestCollections clears passed collections
+func RemoveTestCollections(t *testing.T, c *Connection, collections ...string) {
+ log.Printf("[DEBUG] clean test collections %+v", collections)
+ for _, collection := range collections {
+ _ = c.WithCustomCollection(collection, func(coll *mgo.Collection) error {
+ _, e := coll.RemoveAll(nil)
+ require.Nil(t, e, "failed to remove records, %s", e)
+ e = coll.DropCollection()
+ if e != nil && e.Error() != "ns not found" {
+ require.Nil(t, e, "failed to drop collection, %s", e)
+ }
+ return e
+ })
+ }
+
+}
diff --git a/backend/app/store/engine/mongo/writer.go b/backend/app/store/engine/mongo/writer.go
new file mode 100644
index 00000000..74e9eb50
--- /dev/null
+++ b/backend/app/store/engine/mongo/writer.go
@@ -0,0 +1,151 @@
+package mongo
+
+import (
+ "context"
+ "log"
+ "sync"
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/pkg/errors"
+)
+
+// BufferedWriter defines interface for writes and flush
+type BufferedWriter interface {
+ Write(rec interface{}) error
+ Flush() error
+ Close() error
+}
+
+// BufferedWriterMgo collects records in local buffer and flushes them as filled. Thread safe
+// by default using both DB and collection from provided connection.
+// Collection can be customized by WithCollection method. Optional flush duration to save on interval
+type BufferedWriterMgo struct {
+ connection *Connection
+ bufferSize int
+ collection string
+ flushDuration time.Duration
+
+ ctx context.Context
+ cancel context.CancelFunc
+
+ buffer []interface{}
+ lock sync.Mutex
+ lastWriteTime time.Time
+ once sync.Once
+}
+
+// NewBufferedWriter makes batch writer for given size and connection
+func NewBufferedWriter(size int, connection *Connection) *BufferedWriterMgo {
+ if size == 0 {
+ size = 1
+ }
+ return &BufferedWriterMgo{
+ bufferSize: size,
+ buffer: make([]interface{}, 0, size+1),
+ connection: connection,
+ }
+}
+
+// WithCollection sets custom collection to use with writer
+func (bw *BufferedWriterMgo) WithCollection(collection string) *BufferedWriterMgo {
+ bw.collection = collection
+ return bw
+}
+
+// WithAutoFlush sets auto flush duration
+func (bw *BufferedWriterMgo) WithAutoFlush(duration time.Duration) *BufferedWriterMgo {
+ bw.flushDuration = duration
+ if duration > 0 { // activate background auto-flush
+ bw.once.Do(func() {
+ bw.ctx, bw.cancel = context.WithCancel(context.Background())
+ ticker := time.NewTicker(duration)
+ go func() {
+ defer bw.cancel()
+ for {
+ select {
+ case <-ticker.C:
+ var shouldFlush bool
+ _ = bw.synced(func() error {
+ shouldFlush = time.Now().After(bw.lastWriteTime.Add(bw.flushDuration)) && len(bw.buffer) > 0
+ return nil
+ })
+ if shouldFlush {
+ if err := bw.Flush(); err != nil {
+ log.Printf("[WARN] flush failed, %s", err)
+ }
+ }
+ case <-bw.ctx.Done():
+ log.Printf("[DEBUG] mongo writer flusher terminated")
+ return
+ }
+ }
+ }()
+ })
+ }
+ return bw
+}
+
+// Write to buffer and, as filled, to mongo. If flushDuration defined check for automatic flush
+func (bw *BufferedWriterMgo) Write(rec interface{}) error {
+ return bw.synced(func() error {
+ bw.lastWriteTime = time.Now()
+ bw.buffer = append(bw.buffer, rec)
+ if len(bw.buffer) >= bw.bufferSize {
+ err := bw.writeBuffer()
+ bw.buffer = bw.buffer[0:0]
+ return errors.Wrapf(err, "failed to write to %s", bw.connection)
+ }
+ return nil
+ })
+}
+
+// Flush writes everything left in buffer to mongo
+func (bw *BufferedWriterMgo) Flush() error {
+ return bw.synced(func() error {
+ err := bw.writeBuffer()
+ bw.buffer = bw.buffer[0:0]
+ return errors.Wrapf(err, "failed to flush to %s", bw.connection)
+ })
+}
+
+// Close flushes all in-fly records and terminates background auto-flusher
+func (bw *BufferedWriterMgo) Close() (err error) {
+ return bw.synced(func() error {
+ err = bw.writeBuffer()
+ if bw.flushDuration > 0 {
+ bw.cancel()
+ <-bw.ctx.Done()
+ log.Printf("[DEBUG] mongo buffered writer closed")
+ }
+ return err
+ })
+}
+
+// writeBuffer sends all collected records to mongo
+func (bw *BufferedWriterMgo) writeBuffer() (err error) {
+
+ if len(bw.buffer) == 0 {
+ return nil
+ }
+
+ if bw.collection == "" { // no custom collection
+ err = bw.connection.WithCollection(func(coll *mgo.Collection) error {
+ return coll.Insert(bw.buffer...)
+ })
+ }
+
+ if bw.collection != "" { // with custom collection
+ err = bw.connection.WithCustomCollection(bw.collection, func(coll *mgo.Collection) error {
+ return coll.Insert(bw.buffer...)
+ })
+ }
+
+ return err
+}
+
+func (bw *BufferedWriterMgo) synced(fn func() error) error {
+ bw.lock.Lock()
+ defer bw.lock.Unlock()
+ return fn()
+}
diff --git a/backend/app/store/engine/mongo/writer_test.go b/backend/app/store/engine/mongo/writer_test.go
new file mode 100644
index 00000000..1c4a79f5
--- /dev/null
+++ b/backend/app/store/engine/mongo/writer_test.go
@@ -0,0 +1,162 @@
+package mongo
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/globalsign/mgo"
+ "github.com/globalsign/mgo/bson"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestWriter(t *testing.T) {
+
+ count := func(conn *Connection) (res int) {
+ _ = conn.WithCollection(func(coll *mgo.Collection) error {
+ var err error
+ res, err = coll.Find(nil).Count()
+ assert.Nil(t, err)
+ return nil
+ })
+ return res
+ }
+
+ conn, err := MakeTestConnection(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, conn)
+
+ var wr BufferedWriter = NewBufferedWriter(3, conn)
+ assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1")
+ assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2")
+
+ assert.Equal(t, 0, count(conn), "nothing yet")
+
+ assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3")
+ assert.Equal(t, 3, count(conn), "all 3 records in")
+
+ assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4")
+ assert.Equal(t, 3, count(conn), "still 3 records")
+
+ assert.Nil(t, wr.Flush())
+ assert.Equal(t, 4, count(conn), "all 4 records")
+
+ assert.Nil(t, wr.Flush())
+ assert.Equal(t, 4, count(conn), "still 4 records, nothing left to flush")
+
+ assert.Nil(t, wr.Close())
+
+}
+
+func TestWriter_Parallel(t *testing.T) {
+ conn, err := MakeTestConnection(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, conn)
+
+ var wg sync.WaitGroup
+ wr := NewBufferedWriter(75, conn)
+
+ writeMany := func() {
+ for i := 0; i < 100; i++ {
+ require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2}))
+ }
+ wr.Flush()
+ wg.Done()
+ }
+
+ for i := 0; i < 16; i++ {
+ wg.Add(1)
+ go writeMany()
+ }
+
+ wg.Wait()
+
+ _ = conn.WithCollection(func(coll *mgo.Collection) error {
+ res, err := coll.Find(nil).Count()
+ assert.Nil(t, err)
+ assert.Equal(t, 100*16, res)
+ return nil
+ })
+ assert.Nil(t, wr.Close())
+}
+
+func TestWriter_WithAuthFlush(t *testing.T) {
+ conn, err := MakeTestConnection(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, conn)
+
+ var wr BufferedWriter = NewBufferedWriter(3, conn).WithAutoFlush(500 * time.Millisecond)
+ count := func() (res int) {
+ _ = conn.WithCollection(func(coll *mgo.Collection) error {
+ var err error
+ res, err = coll.Find(nil).Count()
+ assert.Nil(t, err)
+ return nil
+ })
+ return res
+ }
+
+ assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1")
+ assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2")
+ assert.Equal(t, 0, count(), "nothing yet")
+ time.Sleep(600 * time.Millisecond)
+ assert.Equal(t, 2, count(), "2 records flushed")
+
+ assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3")
+ assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4")
+ assert.Nil(t, wr.Write(bson.M{"key5": "val5"}), "write rec #5")
+ assert.Equal(t, 5, count(), "5 records, flushed by size, not duration")
+
+ assert.Nil(t, wr.Write(bson.M{"key6": "val6"}), "write rec #6")
+ assert.Nil(t, wr.Write(bson.M{"key7": "val7"}), "write rec #7")
+ assert.Equal(t, 5, count(), "still 5 records")
+
+ assert.Nil(t, wr.Flush())
+ assert.Equal(t, 7, count(), "all 7 records")
+
+ assert.Nil(t, wr.Flush())
+ assert.Equal(t, 7, count(), "still 7 records, nothing left to flush")
+ assert.Nil(t, wr.Close())
+}
+
+func TestWriter_ParallelWithAutoFlush(t *testing.T) {
+ conn, err := MakeTestConnection(t)
+ if err != nil {
+ return
+ }
+ defer RemoveTestCollection(t, conn)
+
+ var wg sync.WaitGroup
+ wr := NewBufferedWriter(75, conn).WithAutoFlush(time.Millisecond)
+
+ writeMany := func() {
+ for i := 0; i < 100; i++ {
+ require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2}))
+ time.Sleep(time.Millisecond * 3)
+ }
+ wr.Flush()
+ wg.Done()
+ }
+
+ for i := 0; i < 16; i++ {
+ wg.Add(1)
+ go writeMany()
+ }
+
+ wg.Wait()
+
+ _ = conn.WithCollection(func(coll *mgo.Collection) error {
+ res, err := coll.Find(nil).Count()
+ assert.Nil(t, err)
+ assert.Equal(t, 100*16, res)
+ return nil
+ })
+ assert.Nil(t, wr.Close())
+}
diff --git a/backend/app/store/engine/mongo_test.go b/backend/app/store/engine/mongo_test.go
new file mode 100644
index 00000000..c2659c6d
--- /dev/null
+++ b/backend/app/store/engine/mongo_test.go
@@ -0,0 +1,582 @@
+package engine
+
+import (
+ "fmt"
+ "math/rand"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/umputun/remark/backend/app/store"
+ "github.com/umputun/remark/backend/app/store/engine/mongo"
+)
+
+func TestMongo_CreateAndFind(t *testing.T) {
+ var m Interface
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+ assert.Nil(t, err)
+ require.Equal(t, 2, len(res))
+ assert.Equal(t, `some text, link`, res[0].Text)
+ assert.Equal(t, "user1", res[0].User.ID)
+ t.Log(res[0].ID)
+
+ _, err = m.Create(store.Comment{ID: res[0].ID, Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+ assert.NotNil(t, err, "reject dup")
+
+ id, err := m.Create(store.Comment{ID: "id-3", Locator: store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}})
+ assert.Nil(t, err)
+ assert.Equal(t, "id-3", id)
+ res, err = m.Find(store.Locator{URL: "https://radio-t2.com", SiteID: "radio-t2"}, "time")
+ assert.Nil(t, err)
+ require.Equal(t, 1, len(res))
+
+ assert.NoError(t, m.Close())
+}
+
+func TestMongo_Get(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+
+ comment, err := m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, res[1].ID)
+ assert.Nil(t, err)
+ assert.Equal(t, "some text2", comment.Text)
+
+ comment, err = m.Get(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "1234567")
+ assert.NotNil(t, err, "not found")
+}
+
+func TestMongo_Put(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+ res, err := m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+
+ comment := res[0]
+ comment.Text = "abc 123"
+ comment.Score = 100
+ err = m.Put(loc, comment)
+ assert.Nil(t, err)
+
+ comment, err = m.Get(loc, res[0].ID)
+ assert.Nil(t, err)
+ assert.Equal(t, "abc 123", comment.Text)
+ assert.Equal(t, res[0].ID, comment.ID)
+ assert.Equal(t, 100, comment.Score)
+
+ err = m.Put(store.Locator{URL: "https://radio-t.com", SiteID: "bad"}, comment)
+ assert.EqualError(t, err, `not found`)
+
+ err = m.Put(store.Locator{URL: "https://radio-t.com-bad", SiteID: "radio-t"}, comment)
+ assert.EqualError(t, err, `not found`)
+}
+
+func TestMongo_Last(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ res, err := m.Last("radio-t", 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+
+ res, err = m.Last("radio-t", 1)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(res))
+ assert.Equal(t, "some text2", res[0].Text)
+}
+
+func TestMongo_Count(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+ assert.Nil(t, err)
+ assert.Equal(t, 2, c)
+
+ c, err = m.Count(store.Locator{URL: "https://radio-t.com-xxx", SiteID: "radio-t"})
+ assert.Nil(t, err)
+ assert.Equal(t, 0, c)
+}
+
+func TestMongo_List(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := m.Create(comment)
+ assert.Nil(t, err)
+
+ ts := func(sec int) time.Time { return time.Date(2017, 12, 20, 15, 18, sec, 0, time.Local).In(time.UTC) }
+
+ res, err := m.List("radio-t", 0, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}},
+ res)
+
+ res, err = m.List("radio-t", -1, -1)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)},
+ {URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ res, err = m.List("radio-t", 1, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(22), LastTS: ts(22)}}, res)
+
+ res, err = m.List("radio-t", 1, 1)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}}, res)
+
+ res, err = m.List("bad", 1, 1)
+ assert.Nil(t, err)
+ assert.Equal(t, []store.PostInfo{}, res)
+}
+
+func TestMongo_Info(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ ts := func(min int) time.Time { return time.Date(2017, 12, 20, 15, 18, min, 0, time.Local).In(time.UTC) }
+
+ // add one more for https://radio-t.com/2
+ comment := store.Comment{
+ ID: "12345",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 24, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ _, err := m.Create(comment)
+ assert.Nil(t, err)
+
+ r, err := m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
+ require.Nil(t, err)
+ assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24)}, r)
+
+ r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 10)
+ require.Nil(t, err)
+ assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
+
+ r, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, 0)
+ require.Nil(t, err)
+ assert.Equal(t, store.PostInfo{URL: "https://radio-t.com", Count: 2, FirstTS: ts(22), LastTS: ts(23)}, r)
+
+ _, err = m.Info(store.Locator{URL: "https://radio-t.com/error", SiteID: "radio-t"}, 0)
+ require.NotNil(t, err)
+
+ _, err = m.Info(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t-error"}, 0)
+ require.NotNil(t, err)
+
+ err = m.SetReadOnly(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, true)
+ require.Nil(t, err)
+ r, err = m.Info(store.Locator{URL: "https://radio-t.com/2", SiteID: "radio-t"}, 0)
+ require.Nil(t, err)
+ assert.Equal(t, store.PostInfo{URL: "https://radio-t.com/2", Count: 1, FirstTS: ts(24), LastTS: ts(24), ReadOnly: true}, r)
+}
+
+func TestMongo_ReadOnly(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "nothing ro")
+
+ assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, true))
+ assert.True(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 ro")
+
+ assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-2"}), "url-2 still writable")
+
+ assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}, false))
+ assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1"}), "url-1 writable")
+
+ assert.NotNil(t, m.SetReadOnly(store.Locator{SiteID: "bad", URL: "url-1"}, true), "nos site \"bad\"")
+ assert.NoError(t, m.SetReadOnly(store.Locator{SiteID: "radio-t", URL: "url-1xyz"}, false))
+
+ assert.False(t, m.IsReadOnly(store.Locator{SiteID: "radio-t-bad", URL: "url-1"}), "nothing blocked on wrong site")
+}
+
+func TestMongo_Verified(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ assert.False(t, m.IsVerified("radio-t", "u1"), "nothing verified")
+
+ assert.NoError(t, m.SetVerified("radio-t", "u1", true))
+ assert.True(t, m.IsVerified("radio-t", "u1"), "u1 verified")
+
+ assert.False(t, m.IsVerified("radio-t", "u2"), "u2 still not verified")
+ assert.NoError(t, m.SetVerified("radio-t", "u1", false))
+ assert.False(t, m.IsVerified("radio-t", "u1"), "u1 not verified anymore")
+
+ assert.NotNil(t, m.SetVerified("bad", "u1", true), `site "bad" not found`)
+ assert.NoError(t, m.SetVerified("radio-t", "u1xyz", false))
+
+ assert.False(t, m.IsVerified("radio-t-bad", "u1"), "nothing verified on wrong site")
+}
+
+func TestMongo_GetForUser(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ res, err := m.User("radio-t", "user1", 5, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ res, err = m.User("radio-t", "user1", 1, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, "some text2", res[0].Text, "sorted by -time")
+
+ res, err = m.User("radio-t", "user1", 1, 1)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(res), "allow 1 comment")
+ assert.Equal(t, `some text, link`, res[0].Text, "second comment")
+
+ res, err = m.User("bad", "user1", 1, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestMongo_GetForUserPagination(t *testing.T) {
+ m, skip := prepMongo(t, false)
+ if skip {
+ return
+ }
+ c := store.Comment{
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+
+ // write 200 comments
+ for i := 0; i < 200; i++ {
+ c.ID = fmt.Sprintf("id-%d", i)
+ c.Text = fmt.Sprintf("text #%d", i)
+ c.Timestamp = time.Date(2017, 12, 20, 15, 18, i, 0, time.Local)
+ _, err := m.Create(c)
+ require.Nil(t, err, c.ID)
+ }
+
+ // get all comments
+ res, err := m.User("radio-t", "user1", 0, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 200, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+
+ // seek 0, 5 comments
+ res, err = m.User("radio-t", "user1", 5, 0)
+ assert.Nil(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-199", res[0].ID)
+ assert.Equal(t, "id-195", res[4].ID)
+
+ // seek 10, 3 comments
+ res, err = m.User("radio-t", "user1", 3, 10)
+ assert.Nil(t, err)
+ assert.Equal(t, 3, len(res))
+ assert.Equal(t, "id-189", res[0].ID)
+ assert.Equal(t, "id-187", res[2].ID)
+
+ // seek 195, ask 10 comments
+ res, err = m.User("radio-t", "user1", 10, 195)
+ assert.Nil(t, err)
+ assert.Equal(t, 5, len(res))
+ assert.Equal(t, "id-4", res[0].ID)
+ assert.Equal(t, "id-0", res[4].ID)
+
+ // seek 255, ask 10 comments
+ res, err = m.User("radio-t", "user1", 10, 255)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(res))
+}
+
+func TestMongo_BlockUser(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
+
+ assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
+ assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
+
+ assert.False(t, m.IsBlocked("radio-t", "user2"), "user2 still unblocked")
+
+ assert.NoError(t, m.SetBlock("radio-t", "user1", false, 0))
+ assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 unblocked")
+
+ assert.NotNil(t, m.SetBlock("bad", "user1", true, 0), `site "bad" not found`)
+ assert.NoError(t, m.SetBlock("radio-t", "userX", false, 0))
+
+ assert.False(t, m.IsBlocked("radio-t-bad", "user1"), "nothing blocked on wrong site")
+}
+
+func TestMongo_BlockUserWithTTL(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ assert.False(t, m.IsBlocked("radio-t", "user1"), "nothing blocked")
+ assert.NoError(t, m.SetBlock("radio-t", "user1", true, 500*time.Millisecond))
+ assert.True(t, m.IsBlocked("radio-t", "user1"), "user1 blocked")
+ time.Sleep(500 * time.Millisecond)
+ assert.False(t, m.IsBlocked("radio-t", "user1"), "user1 un-blocked automatically")
+}
+
+func TestMongo_GetForUserCounter(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ count, err := m.UserCount("radio-t", "user1")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, count)
+
+ count, err = m.UserCount("bad", "user1")
+ assert.Nil(t, err)
+ assert.Equal(t, 0, count)
+}
+
+func TestMongo_BlockList(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ assert.NoError(t, m.SetBlock("radio-t", "user1", true, 0))
+ assert.NoError(t, m.SetBlock("radio-t", "user2", true, 500*time.Millisecond))
+ assert.NoError(t, m.SetBlock("radio-t", "user3", false, 0))
+
+ ids, err := m.Blocked("radio-t")
+ assert.NoError(t, err)
+
+ assert.Equal(t, 2, len(ids))
+ assert.Equal(t, "user1", ids[0].ID)
+ assert.Equal(t, "user2", ids[1].ID)
+ t.Logf("%+v", ids)
+
+ time.Sleep(500 * time.Millisecond)
+ ids, err = m.Blocked("radio-t")
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(ids))
+ assert.Equal(t, "user1", ids[0].ID)
+
+ ids, err = m.Blocked("bad")
+ assert.NoError(t, err)
+ assert.Equal(t, 0, len(ids))
+}
+
+func TestMongo_Delete(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+ res, err := m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ err = m.Delete(loc, res[0].ID, store.SoftDelete)
+ assert.Nil(t, err)
+
+ res, err = m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "user name", ID: "user1", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+
+ assert.Equal(t, "some text2", res[1].Text)
+ assert.False(t, res[1].Deleted)
+
+ comments, err := m.Last("radio-t", 10)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(comments), "1 in last, 1 removed")
+
+ err = m.Delete(loc, "123456", store.SoftDelete)
+ assert.NotNil(t, err)
+
+ loc.SiteID = "bad"
+ err = m.Delete(loc, res[0].ID, store.SoftDelete)
+ assert.EqualError(t, err, `can't delete id-1: not found`)
+
+ loc = store.Locator{URL: "https://radio-t.com/bad", SiteID: "radio-t"}
+ err = m.Delete(loc, res[0].ID, store.SoftDelete)
+ assert.EqualError(t, err, `can't delete id-1: not found`)
+}
+
+func TestMongo_DeleteHard(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+ res, err := m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ err = m.Delete(loc, res[0].ID, store.HardDelete)
+ assert.Nil(t, err)
+
+ res, err = m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res))
+ assert.Equal(t, "", res[0].Text)
+ assert.True(t, res[0].Deleted, "marked deleted")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+}
+
+func TestMongo_DeleteAll(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+ res, err := m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res), "initially 2 comments")
+
+ err = m.DeleteAll("radio-t")
+ assert.Nil(t, err)
+
+ comments, err := m.Last("radio-t", 10)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+
+ c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+ assert.Nil(t, err)
+ assert.Equal(t, 0, c, "0 count")
+}
+
+func TestMongo_DeleteUser(t *testing.T) {
+ m, skip := prepMongo(t, true) // adds two comments
+ if skip {
+ return
+ }
+ err := m.DeleteUser("radio-t", "user1")
+ require.NoError(t, err)
+
+ loc := store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}
+ res, err := m.Find(loc, "time")
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(res), "2 comments with deleted info")
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[0].User)
+ assert.Equal(t, store.User{Name: "deleted", ID: "deleted", Picture: "", Admin: false, Blocked: false, IP: ""}, res[1].User)
+
+ c, err := m.Count(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"})
+ assert.Nil(t, err)
+ assert.Equal(t, 0, c, "0 count")
+
+ cc, err := m.User("radio-t", "user1", 5, 0)
+ assert.Nil(t, err, "no comments for user user1 in store")
+ assert.Equal(t, 0, len(cc), "no comments for user user1 in store")
+
+ comments, err := m.Last("radio-t", 10)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(comments), "nothing left")
+}
+
+func TestMongo_Parallel(t *testing.T) {
+ var m Interface
+ var skip bool
+ m, skip = prepMongoBuffered(t) // buffered engine, no comments
+ if skip {
+ return
+ }
+ go func() {
+ for i := 0; i < 100; i++ {
+ _, err := m.Create(store.Comment{
+ ID: fmt.Sprintf("id-%d", i), Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}})
+ require.Nil(t, err)
+ time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond)
+ }
+ }()
+
+ for {
+ time.Sleep(10 * time.Millisecond)
+ res, err := m.Find(store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"}, "time")
+ assert.Nil(t, err)
+ if len(res) == 100 {
+ break
+ }
+ }
+}
+
+func prepMongo(t *testing.T, writeRecs bool) (*Mongo, bool) {
+ conn, err := mongo.MakeTestConnection(t)
+ if err != nil {
+ return nil, true
+ }
+ mongo.RemoveTestCollection(t, conn)
+
+ m, err := NewMongo(conn, 1, 0*time.Microsecond)
+ require.Nil(t, err)
+
+ mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
+ comment := store.Comment{
+ ID: "id-1",
+ Text: `some text, link`,
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ if writeRecs {
+ _, err = m.Create(comment)
+ assert.Nil(t, err)
+ }
+
+ comment = store.Comment{
+ ID: "id-2",
+ Text: "some text2",
+ Timestamp: time.Date(2017, 12, 20, 15, 18, 23, 0, time.Local),
+ Locator: store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
+ User: store.User{ID: "user1", Name: "user name"},
+ }
+ if writeRecs {
+ _, err = m.Create(comment)
+ assert.Nil(t, err)
+ }
+
+ return m, false
+}
+
+func prepMongoBuffered(t *testing.T) (*Mongo, bool) {
+ conn, err := mongo.MakeTestConnection(t)
+ if err != nil {
+ return nil, true
+ }
+ mongo.RemoveTestCollection(t, conn)
+
+ m, err := NewMongo(conn, 10, 10*time.Millisecond)
+ mongo.RemoveTestCollections(t, conn, mongoPosts, mongoMetaPosts, mongoMetaUsers)
+
+ require.Nil(t, err)
+ return m, false
+}
diff --git a/backend/remark.rest b/backend/remark.rest
index 94a20dc8..9c8f26dd 100644
--- a/backend/remark.rest
+++ b/backend/remark.rest
@@ -65,16 +65,16 @@ GET {{host}}/api/v1/id/3665976683?site=remark&url={{url}}
GET {{host}}/api/v1/id/a2ddb8d2f65008ee1a1e3af8df0f26beb042309c?site=remark&url=https://radio-t.com/blah1
### get comment by user id
-GET {{host}}/api/v1/comments?site={{site}}&user=github_ef0f706a79cc24b17bbbb374cd234a691d034128&limit=5
+GET {{host}}/api/v1/comments?site={{site}}&user=github_f1fda731dd18fbb388c943599fcae5a213315add&limit=5
### get comment by user id2
-GET {{host}}/api/v1/comments?site=radiot&user=github_0a4349d868946d7841424c9bdd4415629df771e6
+GET {{host}}/api/v1/comments?site={{site}}&user=github_0a4349d868946d7841424c9bdd4415629df771e6
### get count
-GET {{host}}/api/v1/count?site=remark&url={{url}}
+GET {{host}}/api/v1/count?site={{site}}&url={{url}}
### get counts for many
-POST {{host}}/api/v1/counts?site=remark
+POST {{host}}/api/v1/counts?site={{site}}
Content-Type: application/json
[
@@ -90,10 +90,10 @@ GET {{host}}/api/v1/list?site={{site}}&limit=10&skip=5
GET {{host}}/api/v1/config
### block user
-PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site=remark&block=1
+PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=1
### unblock user
-PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site=remark&block=0
+PUT {{host}}/api/v1/admin/user/disqus_grigorybakunov?site={{site}}&block=0
### list blocked user
GET {{host}}/api/v1/admin/blocked?site={{site}}
@@ -105,19 +105,16 @@ DELETE {{host}}/api/v1/admin/comment/3665976683?site={{site}}&url={{url}}
GET {{host}}/api/v1/info?site={{site}}&url=https://radio-t.com/p/2018/05/08/prep-597/
### post rss
-GET {{host}}/api/v1/rss/post?site=={{site}}&url={{url}}
-
-### reply rss
-GET {{host}}/api/v1/rss/reply?site=radiot&user=github_ef0f706a79cc24b17bbbb374cd234a691d034128
+GET {{host}}/api/v1/rss/post?site={{site}}&url={{url}}
### site rss
-PUT {{host}}/api/v1/rss/site?site=remark
+GET {{host}}/api/v1/rss/site?site={{site}}
### get default avatar
GET {{host}}/api/v1/avatar/blah
### get config
-GET {{host}}/api/v1/config?site=remark
+GET {{host}}/api/v1/config?site={{site}}
### ping
GET {{host}}/ping
diff --git a/backend/scripts/restore-backup.sh b/backend/scripts/restore-backup.sh
index f32a231c..a50b0104 100755
--- a/backend/scripts/restore-backup.sh
+++ b/backend/scripts/restore-backup.sh
@@ -5,6 +5,7 @@ BACKUP_PATH=${BACKUP_PATH:-./var}
echo "unpack $1"
gunzip -c ${BACKUP_PATH}/$1 >/tmp/backup.remark
+echo "source file info"
ls -la /tmp/backup.remark
curl -X POST -H "Content-Type: application/json" --data-binary @/tmp/backup.remark "http://127.0.0.1:8081/api/v1/admin/import?site=${2}&provider=native&secret=${SECRET}"
diff --git a/backend/vendor/github.com/globalsign/mgo/.gitignore b/backend/vendor/github.com/globalsign/mgo/.gitignore
new file mode 100644
index 00000000..9a3120f6
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/.gitignore
@@ -0,0 +1,2 @@
+_harness
+.vscode
\ No newline at end of file
diff --git a/backend/vendor/github.com/globalsign/mgo/.travis.yml b/backend/vendor/github.com/globalsign/mgo/.travis.yml
new file mode 100644
index 00000000..78991a8c
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/.travis.yml
@@ -0,0 +1,49 @@
+language: go
+
+go_import_path: github.com/globalsign/mgo
+
+go:
+ - 1.9.x
+ - 1.10.x
+
+env:
+ global:
+ - BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo
+ - FASTDL=https://fastdl.mongodb.org/linux
+ matrix:
+ - MONGODB=x86_64-ubuntu1404-3.0.15
+ - MONGODB=x86_64-ubuntu1404-3.2.17
+ - MONGODB=x86_64-ubuntu1404-3.4.10
+ - MONGODB=x86_64-ubuntu1404-3.6.0
+
+install:
+
+ - wget $FASTDL/mongodb-linux-$MONGODB.tgz
+ - tar xzvf mongodb-linux-$MONGODB.tgz
+ - export PATH=$PWD/mongodb-linux-$MONGODB/bin:$PATH
+
+ - wget $BUCKET/daemontools.tar.gz
+ - tar xzvf daemontools.tar.gz
+ - export PATH=$PWD/daemontools:$PATH
+
+ - go get gopkg.in/check.v1
+ - go get gopkg.in/yaml.v2
+ - go get gopkg.in/tomb.v2
+ - go get github.com/golang/lint
+
+before_script:
+ - golint ./... | grep -v 'ID' | cat
+ - go vet github.com/globalsign/mgo/bson github.com/globalsign/mgo/txn github.com/globalsign/mgo
+ - export NOIPV6=1
+ - make startdb
+
+script:
+ - (cd bson && go test -check.v)
+ - go test -check.v -fast
+ - (cd txn && go test -check.v)
+ - make stopdb
+
+git:
+ depth: 3
+
+# vim:sw=4:ts=4:et
diff --git a/backend/vendor/github.com/globalsign/mgo/CONTRIBUTING.md b/backend/vendor/github.com/globalsign/mgo/CONTRIBUTING.md
new file mode 100644
index 00000000..79539955
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/CONTRIBUTING.md
@@ -0,0 +1,14 @@
+Contributing
+-------------------------
+
+We really appreciate contributions, but they must meet the following requirements:
+
+* A PR should have a brief description of the problem/feature being proposed
+* Pull requests should target the `development` branch
+* Existing tests should pass and any new code should be covered with it's own test(s) (use [travis-ci](https://travis-ci.org))
+* New functions should be [documented](https://blog.golang.org/godoc-documenting-go-code) clearly
+* Code should pass `golint`, `go vet` and `go fmt`
+
+We merge PRs into `development`, which is then tested in a sharded, replicated environment in our datacenter for regressions. Once everyone is happy, we merge to master - this is to maintain a bit of quality control past the usual PR process.
+
+**Thanks** for helping!
diff --git a/backend/vendor/github.com/globalsign/mgo/LICENSE b/backend/vendor/github.com/globalsign/mgo/LICENSE
new file mode 100644
index 00000000..770c7672
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/LICENSE
@@ -0,0 +1,25 @@
+mgo - MongoDB driver for Go
+
+Copyright (c) 2010-2013 - Gustavo Niemeyer
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/backend/vendor/github.com/globalsign/mgo/Makefile b/backend/vendor/github.com/globalsign/mgo/Makefile
new file mode 100644
index 00000000..d1027d45
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/Makefile
@@ -0,0 +1,5 @@
+startdb:
+ @harness/setup.sh start
+
+stopdb:
+ @harness/setup.sh stop
diff --git a/backend/vendor/github.com/globalsign/mgo/README.md b/backend/vendor/github.com/globalsign/mgo/README.md
new file mode 100644
index 00000000..7531fe4e
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/README.md
@@ -0,0 +1,99 @@
+[](https://travis-ci.org/globalsign/mgo) [](https://godoc.org/github.com/globalsign/mgo)
+
+The MongoDB driver for Go
+-------------------------
+
+This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review.
+Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added.
+
+Further PR's (with tests) are welcome, but please maintain backwards compatibility.
+
+Detailed documentation of the API is available at
+[GoDoc](https://godoc.org/github.com/globalsign/mgo).
+
+A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver.
+
+## Changes
+* Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254))
+* Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288))
+* Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373))
+* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56))
+* Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2))
+* Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5))
+* Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7))
+* Support for partial indexes ([details](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51))
+* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464))
+* Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35))
+* Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16))
+* Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899))
+* Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17))
+* Don't panic when handling indexed `int64` fields ([details](https://github.com/go-mgo/mgo/issues/475))
+* Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25))
+* Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28))
+* Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33))
+* Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37), [more](https://github.com/globalsign/mgo/pull/166))
+* Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41))
+* Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42))
+* Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44))
+* GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40))
+* Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49))
+* Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52))
+* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55))
+* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69))
+* Use JSON tags when no explicit BSON are tags set ([details](https://github.com/globalsign/mgo/pull/91))
+* Support [$changeStream](https://docs.mongodb.com/manual/changeStreams/) tailing on 3.6+ ([details](https://github.com/globalsign/mgo/pull/97))
+* Fix deadlock in cluster synchronisation ([details](https://github.com/globalsign/mgo/issues/120))
+* Implement `maxIdleTimeout` for pooled connections ([details](https://github.com/globalsign/mgo/pull/116))
+* Connection pool waiting improvements ([details](https://github.com/globalsign/mgo/pull/115))
+* Fixes BSON encoding for `$in` and friends ([details](https://github.com/globalsign/mgo/pull/128))
+* Add BSON stream encoders ([details](https://github.com/globalsign/mgo/pull/127))
+* Add integer map key support in the BSON encoder ([details](https://github.com/globalsign/mgo/pull/140))
+* Support aggregation [collations](https://docs.mongodb.com/manual/reference/collation/) ([details](https://github.com/globalsign/mgo/pull/144))
+* Support encoding of inline struct references ([details](https://github.com/globalsign/mgo/pull/146))
+* Improved windows test harness ([details](https://github.com/globalsign/mgo/pull/158))
+* Improved type and nil handling in the BSON codec ([details](https://github.com/globalsign/mgo/pull/147/files), [more](https://github.com/globalsign/mgo/pull/181))
+* Separated network read/write timeouts ([details](https://github.com/globalsign/mgo/pull/161))
+* Expanded dial string configuration options ([details](https://github.com/globalsign/mgo/pull/162))
+* Implement MongoTimestamp ([details](https://github.com/globalsign/mgo/pull/171))
+* Support setting `writeConcern` for `findAndModify` operations ([details](https://github.com/globalsign/mgo/pull/185))
+* Add `ssl` to the dial string options ([details](https://github.com/globalsign/mgo/pull/184))
+
+
+---
+
+### Thanks to
+* @aksentyev
+* @bachue
+* @bozaro
+* @BenLubar
+* @carldunham
+* @carter2000
+* @cedric-cordenier
+* @cezarsa
+* @DaytonG
+* @ddspog
+* @drichelson
+* @dvic
+* @eaglerayp
+* @feliixx
+* @fmpwizard
+* @gazoon
+* @gedge
+* @gnawux
+* @idy
+* @jameinel
+* @jefferickson
+* @johnlawsharrison
+* @KJTsanaktsidis
+* @larrycinnabar
+* @mapete94
+* @maxnoel
+* @mcspring
+* @Mei-Zhao
+* @peterdeka
+* @Reenjii
+* @roobre
+* @smoya
+* @steve-gray
+* @tbruyelle
+* @wgallagher
diff --git a/backend/vendor/github.com/globalsign/mgo/auth.go b/backend/vendor/github.com/globalsign/mgo/auth.go
new file mode 100644
index 00000000..75d2ebc3
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/auth.go
@@ -0,0 +1,467 @@
+// mgo - MongoDB driver for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package mgo
+
+import (
+ "crypto/md5"
+ "crypto/sha1"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "sync"
+
+ "github.com/globalsign/mgo/bson"
+ "github.com/globalsign/mgo/internal/scram"
+)
+
+type authCmd struct {
+ Authenticate int
+
+ Nonce string
+ User string
+ Key string
+}
+
+type startSaslCmd struct {
+ StartSASL int `bson:"startSasl"`
+}
+
+type authResult struct {
+ ErrMsg string
+ Ok bool
+}
+
+type getNonceCmd struct {
+ GetNonce int
+}
+
+type getNonceResult struct {
+ Nonce string
+ Err string `bson:"$err"`
+ Code int
+}
+
+type logoutCmd struct {
+ Logout int
+}
+
+type saslCmd struct {
+ Start int `bson:"saslStart,omitempty"`
+ Continue int `bson:"saslContinue,omitempty"`
+ ConversationId int `bson:"conversationId,omitempty"`
+ Mechanism string `bson:"mechanism,omitempty"`
+ Payload []byte
+}
+
+type saslResult struct {
+ Ok bool `bson:"ok"`
+ NotOk bool `bson:"code"` // Server <= 2.3.2 returns ok=1 & code>0 on errors (WTF?)
+ Done bool
+
+ ConversationId int `bson:"conversationId"`
+ Payload []byte
+ ErrMsg string
+}
+
+type saslStepper interface {
+ Step(serverData []byte) (clientData []byte, done bool, err error)
+ Close()
+}
+
+func (socket *mongoSocket) getNonce() (nonce string, err error) {
+ socket.Lock()
+ for socket.cachedNonce == "" && socket.dead == nil {
+ debugf("Socket %p to %s: waiting for nonce", socket, socket.addr)
+ socket.gotNonce.Wait()
+ }
+ if socket.cachedNonce == "mongos" {
+ socket.Unlock()
+ return "", errors.New("Can't authenticate with mongos; see http://j.mp/mongos-auth")
+ }
+ debugf("Socket %p to %s: got nonce", socket, socket.addr)
+ nonce, err = socket.cachedNonce, socket.dead
+ socket.cachedNonce = ""
+ socket.Unlock()
+ if err != nil {
+ nonce = ""
+ }
+ return
+}
+
+func (socket *mongoSocket) resetNonce() {
+ debugf("Socket %p to %s: requesting a new nonce", socket, socket.addr)
+ op := &queryOp{}
+ op.query = &getNonceCmd{GetNonce: 1}
+ op.collection = "admin.$cmd"
+ op.limit = -1
+ op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) {
+ if err != nil {
+ socket.kill(errors.New("getNonce: "+err.Error()), true)
+ return
+ }
+ result := &getNonceResult{}
+ err = bson.Unmarshal(docData, &result)
+ if err != nil {
+ socket.kill(errors.New("Failed to unmarshal nonce: "+err.Error()), true)
+ return
+ }
+ debugf("Socket %p to %s: nonce unmarshalled: %#v", socket, socket.addr, result)
+ if result.Code == 13390 {
+ // mongos doesn't yet support auth (see http://j.mp/mongos-auth)
+ result.Nonce = "mongos"
+ } else if result.Nonce == "" {
+ var msg string
+ if result.Err != "" {
+ msg = fmt.Sprintf("Got an empty nonce: %s (%d)", result.Err, result.Code)
+ } else {
+ msg = "Got an empty nonce"
+ }
+ socket.kill(errors.New(msg), true)
+ return
+ }
+ socket.Lock()
+ if socket.cachedNonce != "" {
+ socket.Unlock()
+ panic("resetNonce: nonce already cached")
+ }
+ socket.cachedNonce = result.Nonce
+ socket.gotNonce.Signal()
+ socket.Unlock()
+ }
+ err := socket.Query(op)
+ if err != nil {
+ socket.kill(errors.New("resetNonce: "+err.Error()), true)
+ }
+}
+
+func (socket *mongoSocket) Login(cred Credential) error {
+ socket.Lock()
+ if cred.Mechanism == "" && socket.serverInfo.MaxWireVersion >= 3 {
+ cred.Mechanism = "SCRAM-SHA-1"
+ }
+ for _, sockCred := range socket.creds {
+ if sockCred == cred {
+ debugf("Socket %p to %s: login: db=%q user=%q (already logged in)", socket, socket.addr, cred.Source, cred.Username)
+ socket.Unlock()
+ return nil
+ }
+ }
+ if socket.dropLogout(cred) {
+ debugf("Socket %p to %s: login: db=%q user=%q (cached)", socket, socket.addr, cred.Source, cred.Username)
+ socket.creds = append(socket.creds, cred)
+ socket.Unlock()
+ return nil
+ }
+ socket.Unlock()
+
+ debugf("Socket %p to %s: login: db=%q user=%q", socket, socket.addr, cred.Source, cred.Username)
+
+ var err error
+ switch cred.Mechanism {
+ case "", "MONGODB-CR", "MONGO-CR": // Name changed to MONGODB-CR in SERVER-8501.
+ err = socket.loginClassic(cred)
+ case "PLAIN":
+ err = socket.loginPlain(cred)
+ case "MONGODB-X509":
+ err = socket.loginX509(cred)
+ default:
+ // Try SASL for everything else, if it is available.
+ err = socket.loginSASL(cred)
+ }
+
+ if err != nil {
+ debugf("Socket %p to %s: login error: %s", socket, socket.addr, err)
+ } else {
+ debugf("Socket %p to %s: login successful", socket, socket.addr)
+ }
+ return err
+}
+
+func (socket *mongoSocket) loginClassic(cred Credential) error {
+ // Note that this only works properly because this function is
+ // synchronous, which means the nonce won't get reset while we're
+ // using it and any other login requests will block waiting for a
+ // new nonce provided in the defer call below.
+ nonce, err := socket.getNonce()
+ if err != nil {
+ return err
+ }
+ defer socket.resetNonce()
+
+ psum := md5.New()
+ psum.Write([]byte(cred.Username + ":mongo:" + cred.Password))
+
+ ksum := md5.New()
+ ksum.Write([]byte(nonce + cred.Username))
+ ksum.Write([]byte(hex.EncodeToString(psum.Sum(nil))))
+
+ key := hex.EncodeToString(ksum.Sum(nil))
+
+ cmd := authCmd{Authenticate: 1, User: cred.Username, Nonce: nonce, Key: key}
+ res := authResult{}
+ return socket.loginRun(cred.Source, &cmd, &res, func() error {
+ if !res.Ok {
+ return errors.New(res.ErrMsg)
+ }
+ socket.Lock()
+ socket.dropAuth(cred.Source)
+ socket.creds = append(socket.creds, cred)
+ socket.Unlock()
+ return nil
+ })
+}
+
+type authX509Cmd struct {
+ Authenticate int
+ User string
+ Mechanism string
+}
+
+func (socket *mongoSocket) loginX509(cred Credential) error {
+ cmd := authX509Cmd{Authenticate: 1, User: cred.Username, Mechanism: "MONGODB-X509"}
+ res := authResult{}
+ return socket.loginRun(cred.Source, &cmd, &res, func() error {
+ if !res.Ok {
+ return errors.New(res.ErrMsg)
+ }
+ socket.Lock()
+ socket.dropAuth(cred.Source)
+ socket.creds = append(socket.creds, cred)
+ socket.Unlock()
+ return nil
+ })
+}
+
+func (socket *mongoSocket) loginPlain(cred Credential) error {
+ cmd := saslCmd{Start: 1, Mechanism: "PLAIN", Payload: []byte("\x00" + cred.Username + "\x00" + cred.Password)}
+ res := authResult{}
+ return socket.loginRun(cred.Source, &cmd, &res, func() error {
+ if !res.Ok {
+ return errors.New(res.ErrMsg)
+ }
+ socket.Lock()
+ socket.dropAuth(cred.Source)
+ socket.creds = append(socket.creds, cred)
+ socket.Unlock()
+ return nil
+ })
+}
+
+func (socket *mongoSocket) loginSASL(cred Credential) error {
+ var sasl saslStepper
+ var err error
+ if cred.Mechanism == "SCRAM-SHA-1" {
+ // SCRAM is handled without external libraries.
+ sasl = saslNewScram(cred)
+ } else if len(cred.ServiceHost) > 0 {
+ sasl, err = saslNew(cred, cred.ServiceHost)
+ } else {
+ sasl, err = saslNew(cred, socket.Server().Addr)
+ }
+ if err != nil {
+ return err
+ }
+ defer sasl.Close()
+
+ // The goal of this logic is to carry a locked socket until the
+ // local SASL step confirms the auth is valid; the socket needs to be
+ // locked so that concurrent action doesn't leave the socket in an
+ // auth state that doesn't reflect the operations that took place.
+ // As a simple case, imagine inverting login=>logout to logout=>login.
+ //
+ // The logic below works because the lock func isn't called concurrently.
+ locked := false
+ lock := func(b bool) {
+ if locked != b {
+ locked = b
+ if b {
+ socket.Lock()
+ } else {
+ socket.Unlock()
+ }
+ }
+ }
+
+ lock(true)
+ defer lock(false)
+
+ start := 1
+ cmd := saslCmd{}
+ res := saslResult{}
+ for {
+ payload, done, err := sasl.Step(res.Payload)
+ if err != nil {
+ return err
+ }
+ if done && res.Done {
+ socket.dropAuth(cred.Source)
+ socket.creds = append(socket.creds, cred)
+ break
+ }
+ lock(false)
+
+ cmd = saslCmd{
+ Start: start,
+ Continue: 1 - start,
+ ConversationId: res.ConversationId,
+ Mechanism: cred.Mechanism,
+ Payload: payload,
+ }
+ start = 0
+ err = socket.loginRun(cred.Source, &cmd, &res, func() error {
+ // See the comment on lock for why this is necessary.
+ lock(true)
+ if !res.Ok || res.NotOk {
+ return fmt.Errorf("server returned error on SASL authentication step: %s", res.ErrMsg)
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ if done && res.Done {
+ socket.dropAuth(cred.Source)
+ socket.creds = append(socket.creds, cred)
+ break
+ }
+ }
+
+ return nil
+}
+
+func saslNewScram(cred Credential) *saslScram {
+ credsum := md5.New()
+ credsum.Write([]byte(cred.Username + ":mongo:" + cred.Password))
+ client := scram.NewClient(sha1.New, cred.Username, hex.EncodeToString(credsum.Sum(nil)))
+ return &saslScram{cred: cred, client: client}
+}
+
+type saslScram struct {
+ cred Credential
+ client *scram.Client
+}
+
+func (s *saslScram) Close() {}
+
+func (s *saslScram) Step(serverData []byte) (clientData []byte, done bool, err error) {
+ more := s.client.Step(serverData)
+ return s.client.Out(), !more, s.client.Err()
+}
+
+func (socket *mongoSocket) loginRun(db string, query, result interface{}, f func() error) error {
+ var mutex sync.Mutex
+ var replyErr error
+ mutex.Lock()
+
+ op := queryOp{}
+ op.query = query
+ op.collection = db + ".$cmd"
+ op.limit = -1
+ op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) {
+ defer mutex.Unlock()
+
+ if err != nil {
+ replyErr = err
+ return
+ }
+
+ err = bson.Unmarshal(docData, result)
+ if err != nil {
+ replyErr = err
+ } else {
+ // Must handle this within the read loop for the socket, so
+ // that concurrent login requests are properly ordered.
+ replyErr = f()
+ }
+ }
+
+ err := socket.Query(&op)
+ if err != nil {
+ return err
+ }
+ mutex.Lock() // Wait.
+ return replyErr
+}
+
+func (socket *mongoSocket) Logout(db string) {
+ socket.Lock()
+ cred, found := socket.dropAuth(db)
+ if found {
+ debugf("Socket %p to %s: logout: db=%q (flagged)", socket, socket.addr, db)
+ socket.logout = append(socket.logout, cred)
+ }
+ socket.Unlock()
+}
+
+func (socket *mongoSocket) LogoutAll() {
+ socket.Lock()
+ if l := len(socket.creds); l > 0 {
+ debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l)
+ socket.logout = append(socket.logout, socket.creds...)
+ socket.creds = socket.creds[0:0]
+ }
+ socket.Unlock()
+}
+
+func (socket *mongoSocket) flushLogout() (ops []interface{}) {
+ socket.Lock()
+ if l := len(socket.logout); l > 0 {
+ debugf("Socket %p to %s: logout all (flushing %d)", socket, socket.addr, l)
+ for i := 0; i != l; i++ {
+ op := queryOp{}
+ op.query = &logoutCmd{1}
+ op.collection = socket.logout[i].Source + ".$cmd"
+ op.limit = -1
+ ops = append(ops, &op)
+ }
+ socket.logout = socket.logout[0:0]
+ }
+ socket.Unlock()
+ return
+}
+
+func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) {
+ for i, sockCred := range socket.creds {
+ if sockCred.Source == db {
+ copy(socket.creds[i:], socket.creds[i+1:])
+ socket.creds = socket.creds[:len(socket.creds)-1]
+ return sockCred, true
+ }
+ }
+ return cred, false
+}
+
+func (socket *mongoSocket) dropLogout(cred Credential) (found bool) {
+ for i, sockCred := range socket.logout {
+ if sockCred == cred {
+ copy(socket.logout[i:], socket.logout[i+1:])
+ socket.logout = socket.logout[:len(socket.logout)-1]
+ return true
+ }
+ }
+ return false
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/LICENSE b/backend/vendor/github.com/globalsign/mgo/bson/LICENSE
new file mode 100644
index 00000000..89032601
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/LICENSE
@@ -0,0 +1,25 @@
+BSON library for Go
+
+Copyright (c) 2010-2012 - Gustavo Niemeyer
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/README.md b/backend/vendor/github.com/globalsign/mgo/bson/README.md
new file mode 100644
index 00000000..5c5819e6
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/README.md
@@ -0,0 +1,12 @@
+[](https://godoc.org/github.com/globalsign/mgo/bson)
+
+An Implementation of BSON for Go
+--------------------------------
+
+Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go.
+
+While the BSON package implements the BSON spec as faithfully as possible, there
+is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the
+`bson` package. The priority is for backwards compatibility for the `mgo`
+driver, though fixes for obviously buggy behaviour is welcome (and features, etc
+behind feature flags).
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/bson.go b/backend/vendor/github.com/globalsign/mgo/bson/bson.go
new file mode 100644
index 00000000..eb87ef62
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/bson.go
@@ -0,0 +1,836 @@
+// BSON library for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Package bson is an implementation of the BSON specification for Go:
+//
+// http://bsonspec.org
+//
+// It was created as part of the mgo MongoDB driver for Go, but is standalone
+// and may be used on its own without the driver.
+package bson
+
+import (
+ "bytes"
+ "crypto/md5"
+ "crypto/rand"
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "reflect"
+ "runtime"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+//go:generate go run bson_corpus_spec_test_generator.go
+
+// --------------------------------------------------------------------------
+// The public API.
+
+// Element types constants from BSON specification.
+const (
+ ElementFloat64 byte = 0x01
+ ElementString byte = 0x02
+ ElementDocument byte = 0x03
+ ElementArray byte = 0x04
+ ElementBinary byte = 0x05
+ Element06 byte = 0x06
+ ElementObjectId byte = 0x07
+ ElementBool byte = 0x08
+ ElementDatetime byte = 0x09
+ ElementNil byte = 0x0A
+ ElementRegEx byte = 0x0B
+ ElementDBPointer byte = 0x0C
+ ElementJavaScriptWithoutScope byte = 0x0D
+ ElementSymbol byte = 0x0E
+ ElementJavaScriptWithScope byte = 0x0F
+ ElementInt32 byte = 0x10
+ ElementTimestamp byte = 0x11
+ ElementInt64 byte = 0x12
+ ElementDecimal128 byte = 0x13
+ ElementMinKey byte = 0xFF
+ ElementMaxKey byte = 0x7F
+
+ BinaryGeneric byte = 0x00
+ BinaryFunction byte = 0x01
+ BinaryBinaryOld byte = 0x02
+ BinaryUUIDOld byte = 0x03
+ BinaryUUID byte = 0x04
+ BinaryMD5 byte = 0x05
+ BinaryUserDefined byte = 0x80
+)
+
+// Getter interface: a value implementing the bson.Getter interface will have its GetBSON
+// method called when the given value has to be marshalled, and the result
+// of this method will be marshaled in place of the actual object.
+//
+// If GetBSON returns return a non-nil error, the marshalling procedure
+// will stop and error out with the provided value.
+type Getter interface {
+ GetBSON() (interface{}, error)
+}
+
+// Setter interface: a value implementing the bson.Setter interface will receive the BSON
+// value via the SetBSON method during unmarshaling, and the object
+// itself will not be changed as usual.
+//
+// If setting the value works, the method should return nil or alternatively
+// bson.ErrSetZero to set the respective field to its zero value (nil for
+// pointer types). If SetBSON returns a value of type bson.TypeError, the
+// BSON value will be omitted from a map or slice being decoded and the
+// unmarshalling will continue. If it returns any other non-nil error, the
+// unmarshalling procedure will stop and error out with the provided value.
+//
+// This interface is generally useful in pointer receivers, since the method
+// will want to change the receiver. A type field that implements the Setter
+// interface doesn't have to be a pointer, though.
+//
+// Unlike the usual behavior, unmarshalling onto a value that implements a
+// Setter interface will NOT reset the value to its zero state. This allows
+// the value to decide by itself how to be unmarshalled.
+//
+// For example:
+//
+// type MyString string
+//
+// func (s *MyString) SetBSON(raw bson.Raw) error {
+// return raw.Unmarshal(s)
+// }
+//
+type Setter interface {
+ SetBSON(raw Raw) error
+}
+
+// ErrSetZero may be returned from a SetBSON method to have the value set to
+// its respective zero value. When used in pointer values, this will set the
+// field to nil rather than to the pre-allocated value.
+var ErrSetZero = errors.New("set to zero")
+
+// M is a convenient alias for a map[string]interface{} map, useful for
+// dealing with BSON in a native way. For instance:
+//
+// bson.M{"a": 1, "b": true}
+//
+// There's no special handling for this type in addition to what's done anyway
+// for an equivalent map type. Elements in the map will be dumped in an
+// undefined ordered. See also the bson.D type for an ordered alternative.
+type M map[string]interface{}
+
+// D represents a BSON document containing ordered elements. For example:
+//
+// bson.D{{"a", 1}, {"b", true}}
+//
+// In some situations, such as when creating indexes for MongoDB, the order in
+// which the elements are defined is important. If the order is not important,
+// using a map is generally more comfortable. See bson.M and bson.RawD.
+type D []DocElem
+
+// DocElem is an element of the bson.D document representation.
+type DocElem struct {
+ Name string
+ Value interface{}
+}
+
+// Map returns a map out of the ordered element name/value pairs in d.
+func (d D) Map() (m M) {
+ m = make(M, len(d))
+ for _, item := range d {
+ m[item.Name] = item.Value
+ }
+ return m
+}
+
+// The Raw type represents raw unprocessed BSON documents and elements.
+// Kind is the kind of element as defined per the BSON specification, and
+// Data is the raw unprocessed data for the respective element.
+// Using this type it is possible to unmarshal or marshal values partially.
+//
+// Relevant documentation:
+//
+// http://bsonspec.org/#/specification
+//
+type Raw struct {
+ Kind byte
+ Data []byte
+}
+
+// RawD represents a BSON document containing raw unprocessed elements.
+// This low-level representation may be useful when lazily processing
+// documents of uncertain content, or when manipulating the raw content
+// documents in general.
+type RawD []RawDocElem
+
+// RawDocElem elements of RawD type.
+type RawDocElem struct {
+ Name string
+ Value Raw
+}
+
+// ObjectId is a unique ID identifying a BSON value. It must be exactly 12 bytes
+// long. MongoDB objects by default have such a property set in their "_id"
+// property.
+//
+// http://www.mongodb.org/display/DOCS/Object+Ids
+type ObjectId string
+
+// ObjectIdHex returns an ObjectId from the provided hex representation.
+// Calling this function with an invalid hex representation will
+// cause a runtime panic. See the IsObjectIdHex function.
+func ObjectIdHex(s string) ObjectId {
+ d, err := hex.DecodeString(s)
+ if err != nil || len(d) != 12 {
+ panic(fmt.Sprintf("invalid input to ObjectIdHex: %q", s))
+ }
+ return ObjectId(d)
+}
+
+// IsObjectIdHex returns whether s is a valid hex representation of
+// an ObjectId. See the ObjectIdHex function.
+func IsObjectIdHex(s string) bool {
+ if len(s) != 24 {
+ return false
+ }
+ _, err := hex.DecodeString(s)
+ return err == nil
+}
+
+// objectIdCounter is atomically incremented when generating a new ObjectId
+// using NewObjectId() function. It's used as a counter part of an id.
+var objectIdCounter = readRandomUint32()
+
+// readRandomUint32 returns a random objectIdCounter.
+func readRandomUint32() uint32 {
+ var b [4]byte
+ _, err := io.ReadFull(rand.Reader, b[:])
+ if err != nil {
+ panic(fmt.Errorf("cannot read random object id: %v", err))
+ }
+ return uint32((uint32(b[0]) << 0) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24))
+}
+
+// machineId stores machine id generated once and used in subsequent calls
+// to NewObjectId function.
+var machineId = readMachineId()
+var processId = os.Getpid()
+
+// readMachineId generates and returns a machine id.
+// If this function fails to get the hostname it will cause a runtime error.
+func readMachineId() []byte {
+ var sum [3]byte
+ id := sum[:]
+ hostname, err1 := os.Hostname()
+ if err1 != nil {
+ _, err2 := io.ReadFull(rand.Reader, id)
+ if err2 != nil {
+ panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2))
+ }
+ return id
+ }
+ hw := md5.New()
+ hw.Write([]byte(hostname))
+ copy(id, hw.Sum(nil))
+ return id
+}
+
+// NewObjectId returns a new unique ObjectId.
+func NewObjectId() ObjectId {
+ var b [12]byte
+ // Timestamp, 4 bytes, big endian
+ binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix()))
+ // Machine, first 3 bytes of md5(hostname)
+ b[4] = machineId[0]
+ b[5] = machineId[1]
+ b[6] = machineId[2]
+ // Pid, 2 bytes, specs don't specify endianness, but we use big endian.
+ b[7] = byte(processId >> 8)
+ b[8] = byte(processId)
+ // Increment, 3 bytes, big endian
+ i := atomic.AddUint32(&objectIdCounter, 1)
+ b[9] = byte(i >> 16)
+ b[10] = byte(i >> 8)
+ b[11] = byte(i)
+ return ObjectId(b[:])
+}
+
+// NewObjectIdWithTime returns a dummy ObjectId with the timestamp part filled
+// with the provided number of seconds from epoch UTC, and all other parts
+// filled with zeroes. It's not safe to insert a document with an id generated
+// by this method, it is useful only for queries to find documents with ids
+// generated before or after the specified timestamp.
+func NewObjectIdWithTime(t time.Time) ObjectId {
+ var b [12]byte
+ binary.BigEndian.PutUint32(b[:4], uint32(t.Unix()))
+ return ObjectId(string(b[:]))
+}
+
+// String returns a hex string representation of the id.
+// Example: ObjectIdHex("4d88e15b60f486e428412dc9").
+func (id ObjectId) String() string {
+ return fmt.Sprintf(`ObjectIdHex("%x")`, string(id))
+}
+
+// Hex returns a hex representation of the ObjectId.
+func (id ObjectId) Hex() string {
+ return hex.EncodeToString([]byte(id))
+}
+
+// MarshalJSON turns a bson.ObjectId into a json.Marshaller.
+func (id ObjectId) MarshalJSON() ([]byte, error) {
+ return []byte(fmt.Sprintf(`"%x"`, string(id))), nil
+}
+
+var nullBytes = []byte("null")
+
+// UnmarshalJSON turns *bson.ObjectId into a json.Unmarshaller.
+func (id *ObjectId) UnmarshalJSON(data []byte) error {
+ if len(data) > 0 && (data[0] == '{' || data[0] == 'O') {
+ var v struct {
+ Id json.RawMessage `json:"$oid"`
+ Func struct {
+ Id json.RawMessage
+ } `json:"$oidFunc"`
+ }
+ err := jdec(data, &v)
+ if err == nil {
+ if len(v.Id) > 0 {
+ data = []byte(v.Id)
+ } else {
+ data = []byte(v.Func.Id)
+ }
+ }
+ }
+ if len(data) == 2 && data[0] == '"' && data[1] == '"' || bytes.Equal(data, nullBytes) {
+ *id = ""
+ return nil
+ }
+ if len(data) != 26 || data[0] != '"' || data[25] != '"' {
+ return fmt.Errorf("invalid ObjectId in JSON: %s", string(data))
+ }
+ var buf [12]byte
+ _, err := hex.Decode(buf[:], data[1:25])
+ if err != nil {
+ return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err)
+ }
+ *id = ObjectId(string(buf[:]))
+ return nil
+}
+
+// MarshalText turns bson.ObjectId into an encoding.TextMarshaler.
+func (id ObjectId) MarshalText() ([]byte, error) {
+ return []byte(fmt.Sprintf("%x", string(id))), nil
+}
+
+// UnmarshalText turns *bson.ObjectId into an encoding.TextUnmarshaler.
+func (id *ObjectId) UnmarshalText(data []byte) error {
+ if len(data) == 1 && data[0] == ' ' || len(data) == 0 {
+ *id = ""
+ return nil
+ }
+ if len(data) != 24 {
+ return fmt.Errorf("invalid ObjectId: %s", data)
+ }
+ var buf [12]byte
+ _, err := hex.Decode(buf[:], data[:])
+ if err != nil {
+ return fmt.Errorf("invalid ObjectId: %s (%s)", data, err)
+ }
+ *id = ObjectId(string(buf[:]))
+ return nil
+}
+
+// Valid returns true if id is valid. A valid id must contain exactly 12 bytes.
+func (id ObjectId) Valid() bool {
+ return len(id) == 12
+}
+
+// byteSlice returns byte slice of id from start to end.
+// Calling this function with an invalid id will cause a runtime panic.
+func (id ObjectId) byteSlice(start, end int) []byte {
+ if len(id) != 12 {
+ panic(fmt.Sprintf("invalid ObjectId: %q", string(id)))
+ }
+ return []byte(string(id)[start:end])
+}
+
+// Time returns the timestamp part of the id.
+// It's a runtime error to call this method with an invalid id.
+func (id ObjectId) Time() time.Time {
+ // First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
+ secs := int64(binary.BigEndian.Uint32(id.byteSlice(0, 4)))
+ return time.Unix(secs, 0)
+}
+
+// Machine returns the 3-byte machine id part of the id.
+// It's a runtime error to call this method with an invalid id.
+func (id ObjectId) Machine() []byte {
+ return id.byteSlice(4, 7)
+}
+
+// Pid returns the process id part of the id.
+// It's a runtime error to call this method with an invalid id.
+func (id ObjectId) Pid() uint16 {
+ return binary.BigEndian.Uint16(id.byteSlice(7, 9))
+}
+
+// Counter returns the incrementing value part of the id.
+// It's a runtime error to call this method with an invalid id.
+func (id ObjectId) Counter() int32 {
+ b := id.byteSlice(9, 12)
+ // Counter is stored as big-endian 3-byte value
+ return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
+}
+
+// The Symbol type is similar to a string and is used in languages with a
+// distinct symbol type.
+type Symbol string
+
+// Now returns the current time with millisecond precision. MongoDB stores
+// timestamps with the same precision, so a Time returned from this method
+// will not change after a roundtrip to the database. That's the only reason
+// why this function exists. Using the time.Now function also works fine
+// otherwise.
+func Now() time.Time {
+ return time.Unix(0, time.Now().UnixNano()/1e6*1e6)
+}
+
+// MongoTimestamp is a special internal type used by MongoDB that for some
+// strange reason has its own datatype defined in BSON.
+type MongoTimestamp int64
+
+// Time returns the time part of ts which is stored with second precision.
+func (ts MongoTimestamp) Time() time.Time {
+ return time.Unix(int64(uint64(ts)>>32), 0)
+}
+
+// Counter returns the counter part of ts.
+func (ts MongoTimestamp) Counter() uint32 {
+ return uint32(ts)
+}
+
+// NewMongoTimestamp creates a timestamp using the given
+// date `t` (with second precision) and counter `c` (unique for `t`).
+//
+// Returns an error if time `t` is not between 1970-01-01T00:00:00Z
+// and 2106-02-07T06:28:15Z (inclusive).
+//
+// Note that two MongoTimestamps should never have the same (time, counter) combination:
+// the caller must ensure the counter `c` is increased if creating multiple MongoTimestamp
+// values for the same time `t` (ignoring fractions of seconds).
+func NewMongoTimestamp(t time.Time, c uint32) (MongoTimestamp, error) {
+ u := t.Unix()
+ if u < 0 || u > math.MaxUint32 {
+ return -1, errors.New("invalid value for time")
+ }
+
+ i := int64(u<<32 | int64(c))
+
+ return MongoTimestamp(i), nil
+}
+
+type orderKey int64
+
+// MaxKey is a special value that compares higher than all other possible BSON
+// values in a MongoDB database.
+var MaxKey = orderKey(1<<63 - 1)
+
+// MinKey is a special value that compares lower than all other possible BSON
+// values in a MongoDB database.
+var MinKey = orderKey(-1 << 63)
+
+type undefined struct{}
+
+// Undefined represents the undefined BSON value.
+var Undefined undefined
+
+// Binary is a representation for non-standard binary values. Any kind should
+// work, but the following are known as of this writing:
+//
+// 0x00 - Generic. This is decoded as []byte(data), not Binary{0x00, data}.
+// 0x01 - Function (!?)
+// 0x02 - Obsolete generic.
+// 0x03 - UUID
+// 0x05 - MD5
+// 0x80 - User defined.
+//
+type Binary struct {
+ Kind byte
+ Data []byte
+}
+
+// RegEx represents a regular expression. The Options field may contain
+// individual characters defining the way in which the pattern should be
+// applied, and must be sorted. Valid options as of this writing are 'i' for
+// case insensitive matching, 'm' for multi-line matching, 'x' for verbose
+// mode, 'l' to make \w, \W, and similar be locale-dependent, 's' for dot-all
+// mode (a '.' matches everything), and 'u' to make \w, \W, and similar match
+// unicode. The value of the Options parameter is not verified before being
+// marshaled into the BSON format.
+type RegEx struct {
+ Pattern string
+ Options string
+}
+
+// JavaScript is a type that holds JavaScript code. If Scope is non-nil, it
+// will be marshaled as a mapping from identifiers to values that may be
+// used when evaluating the provided Code.
+type JavaScript struct {
+ Code string
+ Scope interface{}
+}
+
+// DBPointer refers to a document id in a namespace.
+//
+// This type is deprecated in the BSON specification and should not be used
+// except for backwards compatibility with ancient applications.
+type DBPointer struct {
+ Namespace string
+ Id ObjectId
+}
+
+const initialBufferSize = 64
+
+func handleErr(err *error) {
+ if r := recover(); r != nil {
+ if _, ok := r.(runtime.Error); ok {
+ panic(r)
+ } else if _, ok := r.(externalPanic); ok {
+ panic(r)
+ } else if s, ok := r.(string); ok {
+ *err = errors.New(s)
+ } else if e, ok := r.(error); ok {
+ *err = e
+ } else {
+ panic(r)
+ }
+ }
+}
+
+// Marshal serializes the in value, which may be a map or a struct value.
+// In the case of struct values, only exported fields will be serialized,
+// and the order of serialized fields will match that of the struct itself.
+// The lowercased field name is used as the key for each exported field,
+// but this behavior may be changed using the respective field tag.
+// The tag may also contain flags to tweak the marshalling behavior for
+// the field. The tag formats accepted are:
+//
+// "[][,[,]]"
+//
+// `(...) bson:"[][,[,]]" (...)`
+//
+// The following flags are currently supported:
+//
+// omitempty Only include the field if it's not set to the zero
+// value for the type or to empty slices or maps.
+//
+// minsize Marshal an int64 value as an int32, if that's feasible
+// while preserving the numeric value.
+//
+// inline Inline the field, which must be a struct or a map,
+// causing all of its fields or keys to be processed as if
+// they were part of the outer struct. For maps, keys must
+// not conflict with the bson keys of other struct fields.
+//
+// Some examples:
+//
+// type T struct {
+// A bool
+// B int "myb"
+// C string "myc,omitempty"
+// D string `bson:",omitempty" json:"jsonkey"`
+// E int64 ",minsize"
+// F int64 "myf,omitempty,minsize"
+// }
+//
+func Marshal(in interface{}) (out []byte, err error) {
+ return MarshalBuffer(in, make([]byte, 0, initialBufferSize))
+}
+
+// MarshalBuffer behaves the same way as Marshal, except that instead of
+// allocating a new byte slice it tries to use the received byte slice and
+// only allocates more memory if necessary to fit the marshaled value.
+func MarshalBuffer(in interface{}, buf []byte) (out []byte, err error) {
+ defer handleErr(&err)
+ e := &encoder{buf}
+ e.addDoc(reflect.ValueOf(in))
+ return e.out, nil
+}
+
+// Unmarshal deserializes data from in into the out value. The out value
+// must be a map, a pointer to a struct, or a pointer to a bson.D value.
+// In the case of struct values, only exported fields will be deserialized.
+// The lowercased field name is used as the key for each exported field,
+// but this behavior may be changed using the respective field tag.
+// The tag may also contain flags to tweak the marshalling behavior for
+// the field. The tag formats accepted are:
+//
+// "[][,[,]]"
+//
+// `(...) bson:"[][,[,]]" (...)`
+//
+// The following flags are currently supported during unmarshal (see the
+// Marshal method for other flags):
+//
+// inline Inline the field, which must be a struct or a map.
+// Inlined structs are handled as if its fields were part
+// of the outer struct. An inlined map causes keys that do
+// not match any other struct field to be inserted in the
+// map rather than being discarded as usual.
+//
+// The target field or element types of out may not necessarily match
+// the BSON values of the provided data. The following conversions are
+// made automatically:
+//
+// - Numeric types are converted if at least the integer part of the
+// value would be preserved correctly
+// - Bools are converted to numeric types as 1 or 0
+// - Numeric types are converted to bools as true if not 0 or false otherwise
+// - Binary and string BSON data is converted to a string, array or byte slice
+//
+// If the value would not fit the type and cannot be converted, it's
+// silently skipped.
+//
+// Pointer values are initialized when necessary.
+func Unmarshal(in []byte, out interface{}) (err error) {
+ if raw, ok := out.(*Raw); ok {
+ raw.Kind = 3
+ raw.Data = in
+ return nil
+ }
+ defer handleErr(&err)
+ v := reflect.ValueOf(out)
+ switch v.Kind() {
+ case reflect.Ptr:
+ fallthrough
+ case reflect.Map:
+ d := newDecoder(in)
+ d.readDocTo(v)
+ if d.i < len(d.in) {
+ return errors.New("document is corrupted")
+ }
+ case reflect.Struct:
+ return errors.New("unmarshal can't deal with struct values. Use a pointer")
+ default:
+ return errors.New("unmarshal needs a map or a pointer to a struct")
+ }
+ return nil
+}
+
+// Unmarshal deserializes raw into the out value. If the out value type
+// is not compatible with raw, a *bson.TypeError is returned.
+//
+// See the Unmarshal function documentation for more details on the
+// unmarshalling process.
+func (raw Raw) Unmarshal(out interface{}) (err error) {
+ defer handleErr(&err)
+ v := reflect.ValueOf(out)
+ switch v.Kind() {
+ case reflect.Ptr:
+ v = v.Elem()
+ fallthrough
+ case reflect.Map:
+ d := newDecoder(raw.Data)
+ good := d.readElemTo(v, raw.Kind)
+ if !good {
+ return &TypeError{v.Type(), raw.Kind}
+ }
+ case reflect.Struct:
+ return errors.New("raw Unmarshal can't deal with struct values. Use a pointer")
+ default:
+ return errors.New("raw Unmarshal needs a map or a valid pointer")
+ }
+ return nil
+}
+
+// TypeError store details for type error occuring
+// during unmarshaling
+type TypeError struct {
+ Type reflect.Type
+ Kind byte
+}
+
+func (e *TypeError) Error() string {
+ return fmt.Sprintf("BSON kind 0x%02x isn't compatible with type %s", e.Kind, e.Type.String())
+}
+
+// --------------------------------------------------------------------------
+// Maintain a mapping of keys to structure field indexes
+
+type structInfo struct {
+ FieldsMap map[string]fieldInfo
+ FieldsList []fieldInfo
+ InlineMap int
+ Zero reflect.Value
+}
+
+type fieldInfo struct {
+ Key string
+ Num int
+ OmitEmpty bool
+ MinSize bool
+ Inline []int
+}
+
+var structMap = make(map[reflect.Type]*structInfo)
+var structMapMutex sync.RWMutex
+
+type externalPanic string
+
+func (e externalPanic) String() string {
+ return string(e)
+}
+
+func getStructInfo(st reflect.Type) (*structInfo, error) {
+ structMapMutex.RLock()
+ sinfo, found := structMap[st]
+ structMapMutex.RUnlock()
+ if found {
+ return sinfo, nil
+ }
+ n := st.NumField()
+ fieldsMap := make(map[string]fieldInfo)
+ fieldsList := make([]fieldInfo, 0, n)
+ inlineMap := -1
+ for i := 0; i != n; i++ {
+ field := st.Field(i)
+ if field.PkgPath != "" && !field.Anonymous {
+ continue // Private field
+ }
+
+ info := fieldInfo{Num: i}
+
+ tag := field.Tag.Get("bson")
+
+ // Fall-back to JSON struct tag, if feature flag is set.
+ if tag == "" && useJSONTagFallback {
+ tag = field.Tag.Get("json")
+ }
+
+ // If there's no bson/json tag available.
+ if tag == "" {
+ // If there's no tag, and also no tag: value splits (i.e. no colon)
+ // then assume the entire tag is the value
+ if strings.Index(string(field.Tag), ":") < 0 {
+ tag = string(field.Tag)
+ }
+ }
+
+ if tag == "-" {
+ continue
+ }
+
+ inline := false
+ fields := strings.Split(tag, ",")
+ if len(fields) > 1 {
+ for _, flag := range fields[1:] {
+ switch flag {
+ case "omitempty":
+ info.OmitEmpty = true
+ case "minsize":
+ info.MinSize = true
+ case "inline":
+ inline = true
+ default:
+ msg := fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)
+ panic(externalPanic(msg))
+ }
+ }
+ tag = fields[0]
+ }
+
+ if inline {
+ switch field.Type.Kind() {
+ case reflect.Map:
+ if inlineMap >= 0 {
+ return nil, errors.New("Multiple ,inline maps in struct " + st.String())
+ }
+ if field.Type.Key() != reflect.TypeOf("") {
+ return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String())
+ }
+ inlineMap = info.Num
+ case reflect.Ptr:
+ // allow only pointer to struct
+ if kind := field.Type.Elem().Kind(); kind != reflect.Struct {
+ return nil, errors.New("Option ,inline allows a pointer only to a struct, was given pointer to " + kind.String())
+ }
+
+ field.Type = field.Type.Elem()
+ fallthrough
+ case reflect.Struct:
+ sinfo, err := getStructInfo(field.Type)
+ if err != nil {
+ return nil, err
+ }
+ for _, finfo := range sinfo.FieldsList {
+ if _, found := fieldsMap[finfo.Key]; found {
+ msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String()
+ return nil, errors.New(msg)
+ }
+ if finfo.Inline == nil {
+ finfo.Inline = []int{i, finfo.Num}
+ } else {
+ finfo.Inline = append([]int{i}, finfo.Inline...)
+ }
+ fieldsMap[finfo.Key] = finfo
+ fieldsList = append(fieldsList, finfo)
+ }
+ default:
+ panic("Option ,inline needs a struct value or a pointer to a struct or map field")
+ }
+ continue
+ }
+
+ if tag != "" {
+ info.Key = tag
+ } else {
+ info.Key = strings.ToLower(field.Name)
+ }
+
+ if _, found = fieldsMap[info.Key]; found {
+ msg := "Duplicated key '" + info.Key + "' in struct " + st.String()
+ return nil, errors.New(msg)
+ }
+
+ fieldsList = append(fieldsList, info)
+ fieldsMap[info.Key] = info
+ }
+ sinfo = &structInfo{
+ fieldsMap,
+ fieldsList,
+ inlineMap,
+ reflect.New(st).Elem(),
+ }
+ structMapMutex.Lock()
+ structMap[st] = sinfo
+ structMapMutex.Unlock()
+ return sinfo, nil
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go b/backend/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go
new file mode 100644
index 00000000..3525a004
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go
@@ -0,0 +1,294 @@
+// +build ignore
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "go/format"
+ "html/template"
+ "io/ioutil"
+ "log"
+ "path/filepath"
+ "strings"
+
+ "github.com/globalsign/mgo/internal/json"
+)
+
+func main() {
+ log.SetFlags(0)
+ log.SetPrefix(name + ": ")
+
+ var g Generator
+
+ fmt.Fprintf(&g, "// Code generated by \"%s.go\"; DO NOT EDIT\n\n", name)
+
+ src := g.generate()
+
+ err := ioutil.WriteFile(fmt.Sprintf("%s.go", strings.TrimSuffix(name, "_generator")), src, 0644)
+ if err != nil {
+ log.Fatalf("writing output: %s", err)
+ }
+}
+
+// Generator holds the state of the analysis. Primarily used to buffer
+// the output for format.Source.
+type Generator struct {
+ bytes.Buffer // Accumulated output.
+}
+
+// format returns the gofmt-ed contents of the Generator's buffer.
+func (g *Generator) format() []byte {
+ src, err := format.Source(g.Bytes())
+ if err != nil {
+ // Should never happen, but can arise when developing this code.
+ // The user can compile the output to see the error.
+ log.Printf("warning: internal error: invalid Go generated: %s", err)
+ log.Printf("warning: compile the package to analyze the error")
+ return g.Bytes()
+ }
+ return src
+}
+
+// EVERYTHING ABOVE IS CONSTANT BETWEEN THE GENERATORS
+
+const name = "bson_corpus_spec_test_generator"
+
+func (g *Generator) generate() []byte {
+
+ testFiles, err := filepath.Glob("./specdata/specifications/source/bson-corpus/tests/*.json")
+ if err != nil {
+ log.Fatalf("error reading bson-corpus files: %s", err)
+ }
+
+ tests, err := g.loadTests(testFiles)
+ if err != nil {
+ log.Fatalf("error loading tests: %s", err)
+ }
+
+ tmpl, err := g.getTemplate()
+ if err != nil {
+ log.Fatalf("error loading template: %s", err)
+ }
+
+ tmpl.Execute(&g.Buffer, tests)
+
+ return g.format()
+}
+
+func (g *Generator) loadTests(filenames []string) ([]*testDef, error) {
+ var tests []*testDef
+ for _, filename := range filenames {
+ test, err := g.loadTest(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ tests = append(tests, test)
+ }
+
+ return tests, nil
+}
+
+func (g *Generator) loadTest(filename string) (*testDef, error) {
+ content, err := ioutil.ReadFile(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ var testDef testDef
+ err = json.Unmarshal(content, &testDef)
+ if err != nil {
+ return nil, err
+ }
+
+ names := make(map[string]struct{})
+
+ for i := len(testDef.Valid) - 1; i >= 0; i-- {
+ if testDef.BsonType == "0x05" && testDef.Valid[i].Description == "subtype 0x02" {
+ testDef.Valid = append(testDef.Valid[:i], testDef.Valid[i+1:]...)
+ continue
+ }
+
+ name := cleanupFuncName(testDef.Description + "_" + testDef.Valid[i].Description)
+ nameIdx := name
+ j := 1
+ for {
+ if _, ok := names[nameIdx]; !ok {
+ break
+ }
+
+ nameIdx = fmt.Sprintf("%s_%d", name, j)
+ }
+
+ names[nameIdx] = struct{}{}
+
+ testDef.Valid[i].TestDef = &testDef
+ testDef.Valid[i].Name = nameIdx
+ testDef.Valid[i].StructTest = testDef.TestKey != "" &&
+ (testDef.BsonType != "0x05" || strings.Contains(testDef.Valid[i].Description, "0x00")) &&
+ !testDef.Deprecated
+ }
+
+ for i := len(testDef.DecodeErrors) - 1; i >= 0; i-- {
+ if strings.Contains(testDef.DecodeErrors[i].Description, "UTF-8") {
+ testDef.DecodeErrors = append(testDef.DecodeErrors[:i], testDef.DecodeErrors[i+1:]...)
+ continue
+ }
+
+ name := cleanupFuncName(testDef.Description + "_" + testDef.DecodeErrors[i].Description)
+ nameIdx := name
+ j := 1
+ for {
+ if _, ok := names[nameIdx]; !ok {
+ break
+ }
+
+ nameIdx = fmt.Sprintf("%s_%d", name, j)
+ }
+ names[nameIdx] = struct{}{}
+
+ testDef.DecodeErrors[i].Name = nameIdx
+ }
+
+ return &testDef, nil
+}
+
+func (g *Generator) getTemplate() (*template.Template, error) {
+ content := `package bson_test
+
+import (
+ "encoding/hex"
+ "time"
+
+ . "gopkg.in/check.v1"
+ "github.com/globalsign/mgo/bson"
+)
+
+func testValid(c *C, in []byte, expected []byte, result interface{}) {
+ err := bson.Unmarshal(in, result)
+ c.Assert(err, IsNil)
+
+ out, err := bson.Marshal(result)
+ c.Assert(err, IsNil)
+
+ c.Assert(string(expected), Equals, string(out), Commentf("roundtrip failed for %T, expected '%x' but got '%x'", result, expected, out))
+}
+
+func testDecodeSkip(c *C, in []byte) {
+ err := bson.Unmarshal(in, &struct{}{})
+ c.Assert(err, IsNil)
+}
+
+func testDecodeError(c *C, in []byte, result interface{}) {
+ err := bson.Unmarshal(in, result)
+ c.Assert(err, Not(IsNil))
+}
+
+{{range .}}
+{{range .Valid}}
+func (s *S) Test{{.Name}}(c *C) {
+ b, err := hex.DecodeString("{{.Bson}}")
+ c.Assert(err, IsNil)
+
+ {{if .CanonicalBson}}
+ cb, err := hex.DecodeString("{{.CanonicalBson}}")
+ c.Assert(err, IsNil)
+ {{else}}
+ cb := b
+ {{end}}
+
+ var resultD bson.D
+ testValid(c, b, cb, &resultD)
+ {{if .StructTest}}var resultS struct {
+ Element {{.TestDef.GoType}} ` + "`bson:\"{{.TestDef.TestKey}}\"`" + `
+ }
+ testValid(c, b, cb, &resultS){{end}}
+
+ testDecodeSkip(c, b)
+}
+{{end}}
+
+{{range .DecodeErrors}}
+func (s *S) Test{{.Name}}(c *C) {
+ b, err := hex.DecodeString("{{.Bson}}")
+ c.Assert(err, IsNil)
+
+ var resultD bson.D
+ testDecodeError(c, b, &resultD)
+}
+{{end}}
+{{end}}
+`
+ tmpl, err := template.New("").Parse(content)
+ if err != nil {
+ return nil, err
+ }
+ return tmpl, nil
+}
+
+func cleanupFuncName(name string) string {
+ return strings.Map(func(r rune) rune {
+ if (r >= 48 && r <= 57) || (r >= 65 && r <= 90) || (r >= 97 && r <= 122) {
+ return r
+ }
+ return '_'
+ }, name)
+}
+
+type testDef struct {
+ Description string `json:"description"`
+ BsonType string `json:"bson_type"`
+ TestKey string `json:"test_key"`
+ Valid []*valid `json:"valid"`
+ DecodeErrors []*decodeError `json:"decodeErrors"`
+ Deprecated bool `json:"deprecated"`
+}
+
+func (t *testDef) GoType() string {
+ switch t.BsonType {
+ case "0x01":
+ return "float64"
+ case "0x02":
+ return "string"
+ case "0x03":
+ return "bson.D"
+ case "0x04":
+ return "[]interface{}"
+ case "0x05":
+ return "[]byte"
+ case "0x07":
+ return "bson.ObjectId"
+ case "0x08":
+ return "bool"
+ case "0x09":
+ return "time.Time"
+ case "0x0E":
+ return "string"
+ case "0x10":
+ return "int32"
+ case "0x12":
+ return "int64"
+ case "0x13":
+ return "bson.Decimal"
+ default:
+ return "interface{}"
+ }
+}
+
+type valid struct {
+ Description string `json:"description"`
+ Bson string `json:"bson"`
+ CanonicalBson string `json:"canonical_bson"`
+
+ Name string
+ StructTest bool
+ TestDef *testDef
+}
+
+type decodeError struct {
+ Description string `json:"description"`
+ Bson string `json:"bson"`
+
+ Name string
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/compatibility.go b/backend/vendor/github.com/globalsign/mgo/bson/compatibility.go
new file mode 100644
index 00000000..66efd465
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/compatibility.go
@@ -0,0 +1,29 @@
+package bson
+
+// Current state of the JSON tag fallback option.
+var useJSONTagFallback = false
+var useRespectNilValues = false
+
+// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures
+// without BSON tags on a field will fall-back to using the JSON tag (if present).
+func SetJSONTagFallback(state bool) {
+ useJSONTagFallback = state
+}
+
+// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback
+// for more information.
+func JSONTagFallbackState() bool {
+ return useJSONTagFallback
+}
+
+// SetRespectNilValues enables or disables serializing nil slices or maps to `null` values.
+// In other words it enables `encoding/json` compatible behaviour.
+func SetRespectNilValues(state bool) {
+ useRespectNilValues = state
+}
+
+// RespectNilValuesState returns the current status of the JSON nil slices and maps fallback compatibility option.
+// See SetRespectNilValues for more information.
+func RespectNilValuesState() bool {
+ return useRespectNilValues
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/decimal.go b/backend/vendor/github.com/globalsign/mgo/bson/decimal.go
new file mode 100644
index 00000000..672ba182
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/decimal.go
@@ -0,0 +1,312 @@
+// BSON library for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package bson
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+// Decimal128 holds decimal128 BSON values.
+type Decimal128 struct {
+ h, l uint64
+}
+
+func (d Decimal128) String() string {
+ var pos int // positive sign
+ var e int // exponent
+ var h, l uint64 // significand high/low
+
+ if d.h>>63&1 == 0 {
+ pos = 1
+ }
+
+ switch d.h >> 58 & (1<<5 - 1) {
+ case 0x1F:
+ return "NaN"
+ case 0x1E:
+ return "-Inf"[pos:]
+ }
+
+ l = d.l
+ if d.h>>61&3 == 3 {
+ // Bits: 1*sign 2*ignored 14*exponent 111*significand.
+ // Implicit 0b100 prefix in significand.
+ e = int(d.h>>47&(1<<14-1)) - 6176
+ //h = 4<<47 | d.h&(1<<47-1)
+ // Spec says all of these values are out of range.
+ h, l = 0, 0
+ } else {
+ // Bits: 1*sign 14*exponent 113*significand
+ e = int(d.h>>49&(1<<14-1)) - 6176
+ h = d.h & (1<<49 - 1)
+ }
+
+ // Would be handled by the logic below, but that's trivial and common.
+ if h == 0 && l == 0 && e == 0 {
+ return "-0"[pos:]
+ }
+
+ var repr [48]byte // Loop 5 times over 9 digits plus dot, negative sign, and leading zero.
+ var last = len(repr)
+ var i = len(repr)
+ var dot = len(repr) + e
+ var rem uint32
+Loop:
+ for d9 := 0; d9 < 5; d9++ {
+ h, l, rem = divmod(h, l, 1e9)
+ for d1 := 0; d1 < 9; d1++ {
+ // Handle "-0.0", "0.00123400", "-1.00E-6", "1.050E+3", etc.
+ if i < len(repr) && (dot == i || l == 0 && h == 0 && rem > 0 && rem < 10 && (dot < i-6 || e > 0)) {
+ e += len(repr) - i
+ i--
+ repr[i] = '.'
+ last = i - 1
+ dot = len(repr) // Unmark.
+ }
+ c := '0' + byte(rem%10)
+ rem /= 10
+ i--
+ repr[i] = c
+ // Handle "0E+3", "1E+3", etc.
+ if l == 0 && h == 0 && rem == 0 && i == len(repr)-1 && (dot < i-5 || e > 0) {
+ last = i
+ break Loop
+ }
+ if c != '0' {
+ last = i
+ }
+ // Break early. Works without it, but why.
+ if dot > i && l == 0 && h == 0 && rem == 0 {
+ break Loop
+ }
+ }
+ }
+ repr[last-1] = '-'
+ last--
+
+ if e > 0 {
+ return string(repr[last+pos:]) + "E+" + strconv.Itoa(e)
+ }
+ if e < 0 {
+ return string(repr[last+pos:]) + "E" + strconv.Itoa(e)
+ }
+ return string(repr[last+pos:])
+}
+
+func divmod(h, l uint64, div uint32) (qh, ql uint64, rem uint32) {
+ div64 := uint64(div)
+ a := h >> 32
+ aq := a / div64
+ ar := a % div64
+ b := ar<<32 + h&(1<<32-1)
+ bq := b / div64
+ br := b % div64
+ c := br<<32 + l>>32
+ cq := c / div64
+ cr := c % div64
+ d := cr<<32 + l&(1<<32-1)
+ dq := d / div64
+ dr := d % div64
+ return (aq<<32 | bq), (cq<<32 | dq), uint32(dr)
+}
+
+var dNaN = Decimal128{0x1F << 58, 0}
+var dPosInf = Decimal128{0x1E << 58, 0}
+var dNegInf = Decimal128{0x3E << 58, 0}
+
+func dErr(s string) (Decimal128, error) {
+ return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s)
+}
+
+// ParseDecimal128 parse a string and return the corresponding value as
+// a decimal128
+func ParseDecimal128(s string) (Decimal128, error) {
+ orig := s
+ if s == "" {
+ return dErr(orig)
+ }
+ neg := s[0] == '-'
+ if neg || s[0] == '+' {
+ s = s[1:]
+ }
+
+ if (len(s) == 3 || len(s) == 8) && (s[0] == 'N' || s[0] == 'n' || s[0] == 'I' || s[0] == 'i') {
+ if s == "NaN" || s == "nan" || strings.EqualFold(s, "nan") {
+ return dNaN, nil
+ }
+ if s == "Inf" || s == "inf" || strings.EqualFold(s, "inf") || strings.EqualFold(s, "infinity") {
+ if neg {
+ return dNegInf, nil
+ }
+ return dPosInf, nil
+ }
+ return dErr(orig)
+ }
+
+ var h, l uint64
+ var e int
+
+ var add, ovr uint32
+ var mul uint32 = 1
+ var dot = -1
+ var digits = 0
+ var i = 0
+ for i < len(s) {
+ c := s[i]
+ if mul == 1e9 {
+ h, l, ovr = muladd(h, l, mul, add)
+ mul, add = 1, 0
+ if ovr > 0 || h&((1<<15-1)<<49) > 0 {
+ return dErr(orig)
+ }
+ }
+ if c >= '0' && c <= '9' {
+ i++
+ if c > '0' || digits > 0 {
+ digits++
+ }
+ if digits > 34 {
+ if c == '0' {
+ // Exact rounding.
+ e++
+ continue
+ }
+ return dErr(orig)
+ }
+ mul *= 10
+ add *= 10
+ add += uint32(c - '0')
+ continue
+ }
+ if c == '.' {
+ i++
+ if dot >= 0 || i == 1 && len(s) == 1 {
+ return dErr(orig)
+ }
+ if i == len(s) {
+ break
+ }
+ if s[i] < '0' || s[i] > '9' || e > 0 {
+ return dErr(orig)
+ }
+ dot = i
+ continue
+ }
+ break
+ }
+ if i == 0 {
+ return dErr(orig)
+ }
+ if mul > 1 {
+ h, l, ovr = muladd(h, l, mul, add)
+ if ovr > 0 || h&((1<<15-1)<<49) > 0 {
+ return dErr(orig)
+ }
+ }
+ if dot >= 0 {
+ e += dot - i
+ }
+ if i+1 < len(s) && (s[i] == 'E' || s[i] == 'e') {
+ i++
+ eneg := s[i] == '-'
+ if eneg || s[i] == '+' {
+ i++
+ if i == len(s) {
+ return dErr(orig)
+ }
+ }
+ n := 0
+ for i < len(s) && n < 1e4 {
+ c := s[i]
+ i++
+ if c < '0' || c > '9' {
+ return dErr(orig)
+ }
+ n *= 10
+ n += int(c - '0')
+ }
+ if eneg {
+ n = -n
+ }
+ e += n
+ for e < -6176 {
+ // Subnormal.
+ var div uint32 = 1
+ for div < 1e9 && e < -6176 {
+ div *= 10
+ e++
+ }
+ var rem uint32
+ h, l, rem = divmod(h, l, div)
+ if rem > 0 {
+ return dErr(orig)
+ }
+ }
+ for e > 6111 {
+ // Clamped.
+ var mul uint32 = 1
+ for mul < 1e9 && e > 6111 {
+ mul *= 10
+ e--
+ }
+ h, l, ovr = muladd(h, l, mul, 0)
+ if ovr > 0 || h&((1<<15-1)<<49) > 0 {
+ return dErr(orig)
+ }
+ }
+ if e < -6176 || e > 6111 {
+ return dErr(orig)
+ }
+ }
+
+ if i < len(s) {
+ return dErr(orig)
+ }
+
+ h |= uint64(e+6176) & uint64(1<<14-1) << 49
+ if neg {
+ h |= 1 << 63
+ }
+ return Decimal128{h, l}, nil
+}
+
+func muladd(h, l uint64, mul uint32, add uint32) (resh, resl uint64, overflow uint32) {
+ mul64 := uint64(mul)
+ a := mul64 * (l & (1<<32 - 1))
+ b := a>>32 + mul64*(l>>32)
+ c := b>>32 + mul64*(h&(1<<32-1))
+ d := c>>32 + mul64*(h>>32)
+
+ a = a&(1<<32-1) + uint64(add)
+ b = b&(1<<32-1) + a>>32
+ c = c&(1<<32-1) + b>>32
+ d = d&(1<<32-1) + c>>32
+
+ return (d<<32 | c&(1<<32-1)), (b<<32 | a&(1<<32-1)), uint32(d >> 32)
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/decode.go b/backend/vendor/github.com/globalsign/mgo/bson/decode.go
new file mode 100644
index 00000000..658856ad
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/decode.go
@@ -0,0 +1,1055 @@
+// BSON library for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// gobson - BSON library for Go.
+
+package bson
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net/url"
+ "reflect"
+ "strconv"
+ "sync"
+ "time"
+)
+
+type decoder struct {
+ in []byte
+ i int
+ docType reflect.Type
+}
+
+var typeM = reflect.TypeOf(M{})
+
+func newDecoder(in []byte) *decoder {
+ return &decoder{in, 0, typeM}
+}
+
+// --------------------------------------------------------------------------
+// Some helper functions.
+
+func corrupted() {
+ panic("Document is corrupted")
+}
+
+// --------------------------------------------------------------------------
+// Unmarshaling of documents.
+
+const (
+ setterUnknown = iota
+ setterNone
+ setterType
+ setterAddr
+)
+
+var setterStyles map[reflect.Type]int
+var setterIface reflect.Type
+var setterMutex sync.RWMutex
+
+func init() {
+ var iface Setter
+ setterIface = reflect.TypeOf(&iface).Elem()
+ setterStyles = make(map[reflect.Type]int)
+}
+
+func setterStyle(outt reflect.Type) int {
+ setterMutex.RLock()
+ style := setterStyles[outt]
+ setterMutex.RUnlock()
+ if style != setterUnknown {
+ return style
+ }
+
+ setterMutex.Lock()
+ defer setterMutex.Unlock()
+ if outt.Implements(setterIface) {
+ style = setterType
+ } else if reflect.PtrTo(outt).Implements(setterIface) {
+ style = setterAddr
+ } else {
+ style = setterNone
+ }
+ setterStyles[outt] = style
+ return style
+}
+
+func getSetter(outt reflect.Type, out reflect.Value) Setter {
+ style := setterStyle(outt)
+ if style == setterNone {
+ return nil
+ }
+ if style == setterAddr {
+ if !out.CanAddr() {
+ return nil
+ }
+ out = out.Addr()
+ } else if outt.Kind() == reflect.Ptr && out.IsNil() {
+ out.Set(reflect.New(outt.Elem()))
+ }
+ return out.Interface().(Setter)
+}
+
+func clearMap(m reflect.Value) {
+ var none reflect.Value
+ for _, k := range m.MapKeys() {
+ m.SetMapIndex(k, none)
+ }
+}
+
+func (d *decoder) readDocTo(out reflect.Value) {
+ var elemType reflect.Type
+ outt := out.Type()
+ outk := outt.Kind()
+
+ for {
+ if outk == reflect.Ptr && out.IsNil() {
+ out.Set(reflect.New(outt.Elem()))
+ }
+ if setter := getSetter(outt, out); setter != nil {
+ raw := d.readRaw(ElementDocument)
+ err := setter.SetBSON(raw)
+ if _, ok := err.(*TypeError); err != nil && !ok {
+ panic(err)
+ }
+ return
+ }
+ if outk == reflect.Ptr {
+ out = out.Elem()
+ outt = out.Type()
+ outk = out.Kind()
+ continue
+ }
+ break
+ }
+
+ var fieldsMap map[string]fieldInfo
+ var inlineMap reflect.Value
+ if outt == typeRaw {
+ out.Set(reflect.ValueOf(d.readRaw(ElementDocument)))
+ return
+ }
+
+ origout := out
+ if outk == reflect.Interface {
+ if d.docType.Kind() == reflect.Map {
+ mv := reflect.MakeMap(d.docType)
+ out.Set(mv)
+ out = mv
+ } else {
+ dv := reflect.New(d.docType).Elem()
+ out.Set(dv)
+ out = dv
+ }
+ outt = out.Type()
+ outk = outt.Kind()
+ }
+
+ docType := d.docType
+ keyType := typeString
+ convertKey := false
+ switch outk {
+ case reflect.Map:
+ keyType = outt.Key()
+ if keyType != typeString {
+ convertKey = true
+ }
+ elemType = outt.Elem()
+ if elemType == typeIface {
+ d.docType = outt
+ }
+ if out.IsNil() {
+ out.Set(reflect.MakeMap(out.Type()))
+ } else if out.Len() > 0 {
+ clearMap(out)
+ }
+ case reflect.Struct:
+ sinfo, err := getStructInfo(out.Type())
+ if err != nil {
+ panic(err)
+ }
+ fieldsMap = sinfo.FieldsMap
+ out.Set(sinfo.Zero)
+ if sinfo.InlineMap != -1 {
+ inlineMap = out.Field(sinfo.InlineMap)
+ if !inlineMap.IsNil() && inlineMap.Len() > 0 {
+ clearMap(inlineMap)
+ }
+ elemType = inlineMap.Type().Elem()
+ if elemType == typeIface {
+ d.docType = inlineMap.Type()
+ }
+ }
+ case reflect.Slice:
+ switch outt.Elem() {
+ case typeDocElem:
+ origout.Set(d.readDocElems(outt))
+ return
+ case typeRawDocElem:
+ origout.Set(d.readRawDocElems(outt))
+ return
+ }
+ fallthrough
+ default:
+ panic("Unsupported document type for unmarshalling: " + out.Type().String())
+ }
+
+ end := int(d.readInt32())
+ end += d.i - 4
+ if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' {
+ corrupted()
+ }
+ for d.in[d.i] != '\x00' {
+ kind := d.readByte()
+ name := d.readCStr()
+ if d.i >= end {
+ corrupted()
+ }
+
+ switch outk {
+ case reflect.Map:
+ e := reflect.New(elemType).Elem()
+ if d.readElemTo(e, kind) {
+ k := reflect.ValueOf(name)
+ if convertKey {
+ mapKeyType := out.Type().Key()
+ mapKeyKind := mapKeyType.Kind()
+
+ switch mapKeyKind {
+ case reflect.Int:
+ fallthrough
+ case reflect.Int8:
+ fallthrough
+ case reflect.Int16:
+ fallthrough
+ case reflect.Int32:
+ fallthrough
+ case reflect.Int64:
+ fallthrough
+ case reflect.Uint:
+ fallthrough
+ case reflect.Uint8:
+ fallthrough
+ case reflect.Uint16:
+ fallthrough
+ case reflect.Uint32:
+ fallthrough
+ case reflect.Uint64:
+ fallthrough
+ case reflect.Float32:
+ fallthrough
+ case reflect.Float64:
+ parsed := d.parseMapKeyAsFloat(k, mapKeyKind)
+ k = reflect.ValueOf(parsed)
+ case reflect.String:
+ mapKeyType = keyType
+ default:
+ panic("BSON map must have string or decimal keys. Got: " + outt.String())
+ }
+
+ k = k.Convert(mapKeyType)
+ }
+ out.SetMapIndex(k, e)
+ }
+ case reflect.Struct:
+ if info, ok := fieldsMap[name]; ok {
+ if info.Inline == nil {
+ d.readElemTo(out.Field(info.Num), kind)
+ } else {
+ d.readElemTo(out.FieldByIndex(info.Inline), kind)
+ }
+ } else if inlineMap.IsValid() {
+ if inlineMap.IsNil() {
+ inlineMap.Set(reflect.MakeMap(inlineMap.Type()))
+ }
+ e := reflect.New(elemType).Elem()
+ if d.readElemTo(e, kind) {
+ inlineMap.SetMapIndex(reflect.ValueOf(name), e)
+ }
+ } else {
+ d.dropElem(kind)
+ }
+ case reflect.Slice:
+ }
+
+ if d.i >= end {
+ corrupted()
+ }
+ }
+ d.i++ // '\x00'
+ if d.i != end {
+ corrupted()
+ }
+ d.docType = docType
+}
+
+func (decoder) parseMapKeyAsFloat(k reflect.Value, mapKeyKind reflect.Kind) float64 {
+ parsed, err := strconv.ParseFloat(k.String(), 64)
+ if err != nil {
+ panic("Map key is defined to be a decimal type (" + mapKeyKind.String() + ") but got error " +
+ err.Error())
+ }
+
+ return parsed
+}
+
+func (d *decoder) readArrayDocTo(out reflect.Value) {
+ end := int(d.readInt32())
+ end += d.i - 4
+ if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' {
+ corrupted()
+ }
+ i := 0
+ l := out.Len()
+ for d.in[d.i] != '\x00' {
+ if i >= l {
+ panic("Length mismatch on array field")
+ }
+ kind := d.readByte()
+ for d.i < end && d.in[d.i] != '\x00' {
+ d.i++
+ }
+ if d.i >= end {
+ corrupted()
+ }
+ d.i++
+ d.readElemTo(out.Index(i), kind)
+ if d.i >= end {
+ corrupted()
+ }
+ i++
+ }
+ if i != l {
+ panic("Length mismatch on array field")
+ }
+ d.i++ // '\x00'
+ if d.i != end {
+ corrupted()
+ }
+}
+
+func (d *decoder) readSliceDoc(t reflect.Type) interface{} {
+ tmp := make([]reflect.Value, 0, 8)
+ elemType := t.Elem()
+ if elemType == typeRawDocElem {
+ d.dropElem(ElementArray)
+ return reflect.Zero(t).Interface()
+ }
+ if elemType == typeRaw {
+ return d.readSliceOfRaw()
+ }
+
+ end := int(d.readInt32())
+ end += d.i - 4
+ if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' {
+ corrupted()
+ }
+ for d.in[d.i] != '\x00' {
+ kind := d.readByte()
+ for d.i < end && d.in[d.i] != '\x00' {
+ d.i++
+ }
+ if d.i >= end {
+ corrupted()
+ }
+ d.i++
+ e := reflect.New(elemType).Elem()
+ if d.readElemTo(e, kind) {
+ tmp = append(tmp, e)
+ }
+ if d.i >= end {
+ corrupted()
+ }
+ }
+ d.i++ // '\x00'
+ if d.i != end {
+ corrupted()
+ }
+
+ n := len(tmp)
+ slice := reflect.MakeSlice(t, n, n)
+ for i := 0; i != n; i++ {
+ slice.Index(i).Set(tmp[i])
+ }
+ return slice.Interface()
+}
+
+func BSONElementSize(kind byte, offset int, buffer []byte) (int, error) {
+ switch kind {
+ case ElementFloat64: // Float64
+ return 8, nil
+ case ElementJavaScriptWithoutScope: // JavaScript without scope
+ fallthrough
+ case ElementSymbol: // Symbol
+ fallthrough
+ case ElementString: // UTF-8 string
+ size, err := getSize(offset, buffer)
+ if err != nil {
+ return 0, err
+ }
+ if size < 1 {
+ return 0, errors.New("String size can't be less then one byte")
+ }
+ size += 4
+ if offset+size > len(buffer) {
+ return 0, io.ErrUnexpectedEOF
+ }
+ if buffer[offset+size-1] != 0 {
+ return 0, errors.New("Invalid string: non zero-terminated")
+ }
+ return size, nil
+ case ElementArray: // Array
+ fallthrough
+ case ElementDocument: // Document
+ size, err := getSize(offset, buffer)
+ if err != nil {
+ return 0, err
+ }
+ if size < 5 {
+ return 0, errors.New("Declared document size is too small")
+ }
+ return size, nil
+ case ElementBinary: // Binary
+ size, err := getSize(offset, buffer)
+ if err != nil {
+ return 0, err
+ }
+ if size < 0 {
+ return 0, errors.New("Binary data size can't be negative")
+ }
+ return size + 5, nil
+ case Element06: // Undefined (obsolete, but still seen in the wild)
+ return 0, nil
+ case ElementObjectId: // ObjectId
+ return 12, nil
+ case ElementBool: // Bool
+ return 1, nil
+ case ElementDatetime: // Timestamp
+ return 8, nil
+ case ElementNil: // Nil
+ return 0, nil
+ case ElementRegEx: // RegEx
+ end := offset
+ for i := 0; i < 2; i++ {
+ for end < len(buffer) && buffer[end] != '\x00' {
+ end++
+ }
+ end++
+ }
+ if end > len(buffer) {
+ return 0, io.ErrUnexpectedEOF
+ }
+ return end - offset, nil
+ case ElementDBPointer: // DBPointer
+ size, err := getSize(offset, buffer)
+ if err != nil {
+ return 0, err
+ }
+ if size < 1 {
+ return 0, errors.New("String size can't be less then one byte")
+ }
+ return size + 12 + 4, nil
+ case ElementJavaScriptWithScope: // JavaScript with scope
+ size, err := getSize(offset, buffer)
+ if err != nil {
+ return 0, err
+ }
+ if size < 4+5+5 {
+ return 0, errors.New("Declared document element is too small")
+ }
+ return size, nil
+ case ElementInt32: // Int32
+ return 4, nil
+ case ElementTimestamp: // Mongo-specific timestamp
+ return 8, nil
+ case ElementInt64: // Int64
+ return 8, nil
+ case ElementDecimal128: // Decimal128
+ return 16, nil
+ case ElementMaxKey: // Max key
+ return 0, nil
+ case ElementMinKey: // Min key
+ return 0, nil
+ default:
+ return 0, errors.New(fmt.Sprintf("Unknown element kind (0x%02X)", kind))
+ }
+}
+
+func (d *decoder) readRaw(kind byte) Raw {
+ size, err := BSONElementSize(kind, d.i, d.in)
+ if err != nil {
+ corrupted()
+ }
+ if d.i+size > len(d.in) {
+ corrupted()
+ }
+ d.i += size
+ return Raw{
+ Kind: kind,
+ Data: d.in[d.i-size : d.i],
+ }
+}
+
+func (d *decoder) readSliceOfRaw() interface{} {
+ tmp := make([]Raw, 0, 8)
+ end := int(d.readInt32())
+ end += d.i - 4
+ if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' {
+ corrupted()
+ }
+ for d.in[d.i] != '\x00' {
+ kind := d.readByte()
+ for d.i < end && d.in[d.i] != '\x00' {
+ d.i++
+ }
+ if d.i >= end {
+ corrupted()
+ }
+ d.i++
+ e := d.readRaw(kind)
+ tmp = append(tmp, e)
+ if d.i >= end {
+ corrupted()
+ }
+ }
+ d.i++ // '\x00'
+ if d.i != end {
+ corrupted()
+ }
+ return tmp
+}
+
+var typeSlice = reflect.TypeOf([]interface{}{})
+var typeIface = typeSlice.Elem()
+
+func (d *decoder) readDocElems(typ reflect.Type) reflect.Value {
+ docType := d.docType
+ d.docType = typ
+ slice := make([]DocElem, 0, 8)
+ d.readDocWith(func(kind byte, name string) {
+ e := DocElem{Name: name}
+ v := reflect.ValueOf(&e.Value)
+ if d.readElemTo(v.Elem(), kind) {
+ slice = append(slice, e)
+ }
+ })
+ slicev := reflect.New(typ).Elem()
+ slicev.Set(reflect.ValueOf(slice))
+ d.docType = docType
+ return slicev
+}
+
+func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value {
+ docType := d.docType
+ d.docType = typ
+ slice := make([]RawDocElem, 0, 8)
+ d.readDocWith(func(kind byte, name string) {
+ e := RawDocElem{Name: name, Value: d.readRaw(kind)}
+ slice = append(slice, e)
+ })
+ slicev := reflect.New(typ).Elem()
+ slicev.Set(reflect.ValueOf(slice))
+ d.docType = docType
+ return slicev
+}
+
+func (d *decoder) readDocWith(f func(kind byte, name string)) {
+ end := int(d.readInt32())
+ end += d.i - 4
+ if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' {
+ corrupted()
+ }
+ for d.in[d.i] != '\x00' {
+ kind := d.readByte()
+ name := d.readCStr()
+ if d.i >= end {
+ corrupted()
+ }
+ f(kind, name)
+ if d.i >= end {
+ corrupted()
+ }
+ }
+ d.i++ // '\x00'
+ if d.i != end {
+ corrupted()
+ }
+}
+
+// --------------------------------------------------------------------------
+// Unmarshaling of individual elements within a document.
+func (d *decoder) dropElem(kind byte) {
+ size, err := BSONElementSize(kind, d.i, d.in)
+ if err != nil {
+ corrupted()
+ }
+ if d.i+size > len(d.in) {
+ corrupted()
+ }
+ d.i += size
+}
+
+// Attempt to decode an element from the document and put it into out.
+// If the types are not compatible, the returned ok value will be
+// false and out will be unchanged.
+func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) {
+ outt := out.Type()
+
+ if outt == typeRaw {
+ out.Set(reflect.ValueOf(d.readRaw(kind)))
+ return true
+ }
+
+ if outt == typeRawPtr {
+ raw := d.readRaw(kind)
+ out.Set(reflect.ValueOf(&raw))
+ return true
+ }
+
+ if kind == ElementDocument {
+ // Delegate unmarshaling of documents.
+ outt := out.Type()
+ outk := out.Kind()
+ switch outk {
+ case reflect.Interface, reflect.Ptr, reflect.Struct, reflect.Map:
+ d.readDocTo(out)
+ return true
+ }
+ if setterStyle(outt) != setterNone {
+ d.readDocTo(out)
+ return true
+ }
+ if outk == reflect.Slice {
+ switch outt.Elem() {
+ case typeDocElem:
+ out.Set(d.readDocElems(outt))
+ case typeRawDocElem:
+ out.Set(d.readRawDocElems(outt))
+ default:
+ d.dropElem(kind)
+ }
+ return true
+ }
+ d.dropElem(kind)
+ return true
+ }
+
+ if setter := getSetter(outt, out); setter != nil {
+ err := setter.SetBSON(d.readRaw(kind))
+ if err == ErrSetZero {
+ out.Set(reflect.Zero(outt))
+ return true
+ }
+ if err == nil {
+ return true
+ }
+ if _, ok := err.(*TypeError); !ok {
+ panic(err)
+ }
+ return false
+ }
+
+ var in interface{}
+
+ switch kind {
+ case ElementFloat64:
+ in = d.readFloat64()
+ case ElementString:
+ in = d.readStr()
+ case ElementDocument:
+ panic("Can't happen. Handled above.")
+ case ElementArray:
+ outt := out.Type()
+ if setterStyle(outt) != setterNone {
+ // Skip the value so its data is handed to the setter below.
+ d.dropElem(kind)
+ break
+ }
+ for outt.Kind() == reflect.Ptr {
+ outt = outt.Elem()
+ }
+ switch outt.Kind() {
+ case reflect.Array:
+ d.readArrayDocTo(out)
+ return true
+ case reflect.Slice:
+ in = d.readSliceDoc(outt)
+ default:
+ in = d.readSliceDoc(typeSlice)
+ }
+ case ElementBinary:
+ b := d.readBinary()
+ if b.Kind == BinaryGeneric || b.Kind == BinaryBinaryOld {
+ in = b.Data
+ } else {
+ in = b
+ }
+ case Element06: // Undefined (obsolete, but still seen in the wild)
+ in = Undefined
+ case ElementObjectId:
+ in = ObjectId(d.readBytes(12))
+ case ElementBool:
+ in = d.readBool()
+ case ElementDatetime: // Timestamp
+ // MongoDB handles timestamps as milliseconds.
+ i := d.readInt64()
+ if i == -62135596800000 {
+ in = time.Time{} // In UTC for convenience.
+ } else {
+ in = time.Unix(i/1e3, i%1e3*1e6).UTC()
+ }
+ case ElementNil:
+ in = nil
+ case ElementRegEx:
+ in = d.readRegEx()
+ case ElementDBPointer:
+ in = DBPointer{Namespace: d.readStr(), Id: ObjectId(d.readBytes(12))}
+ case ElementJavaScriptWithoutScope:
+ in = JavaScript{Code: d.readStr()}
+ case ElementSymbol:
+ in = Symbol(d.readStr())
+ case ElementJavaScriptWithScope:
+ start := d.i
+ l := int(d.readInt32())
+ js := JavaScript{d.readStr(), make(M)}
+ d.readDocTo(reflect.ValueOf(js.Scope))
+ if d.i != start+l {
+ corrupted()
+ }
+ in = js
+ case ElementInt32:
+ in = int(d.readInt32())
+ case ElementTimestamp: // Mongo-specific timestamp
+ in = MongoTimestamp(d.readInt64())
+ case ElementInt64:
+ switch out.Type() {
+ case typeTimeDuration:
+ in = time.Duration(time.Duration(d.readInt64()) * time.Millisecond)
+ default:
+ in = d.readInt64()
+ }
+ case ElementDecimal128:
+ in = Decimal128{
+ l: uint64(d.readInt64()),
+ h: uint64(d.readInt64()),
+ }
+ case ElementMaxKey:
+ in = MaxKey
+ case ElementMinKey:
+ in = MinKey
+ default:
+ panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind))
+ }
+
+ if in == nil {
+ out.Set(reflect.Zero(outt))
+ return true
+ }
+
+ outk := outt.Kind()
+
+ // Dereference and initialize pointer if necessary.
+ first := true
+ for outk == reflect.Ptr {
+ if !out.IsNil() {
+ out = out.Elem()
+ } else {
+ elem := reflect.New(outt.Elem())
+ if first {
+ // Only set if value is compatible.
+ first = false
+ defer func(out, elem reflect.Value) {
+ if good {
+ out.Set(elem)
+ }
+ }(out, elem)
+ } else {
+ out.Set(elem)
+ }
+ out = elem
+ }
+ outt = out.Type()
+ outk = outt.Kind()
+ }
+
+ inv := reflect.ValueOf(in)
+ if outt == inv.Type() {
+ out.Set(inv)
+ return true
+ }
+
+ switch outk {
+ case reflect.Interface:
+ out.Set(inv)
+ return true
+ case reflect.String:
+ switch inv.Kind() {
+ case reflect.String:
+ out.SetString(inv.String())
+ return true
+ case reflect.Slice:
+ if b, ok := in.([]byte); ok {
+ out.SetString(string(b))
+ return true
+ }
+ case reflect.Int, reflect.Int64:
+ if outt == typeJSONNumber {
+ out.SetString(strconv.FormatInt(inv.Int(), 10))
+ return true
+ }
+ case reflect.Float64:
+ if outt == typeJSONNumber {
+ out.SetString(strconv.FormatFloat(inv.Float(), 'f', -1, 64))
+ return true
+ }
+ }
+ case reflect.Slice, reflect.Array:
+ // Remember, array (0x04) slices are built with the correct
+ // element type. If we are here, must be a cross BSON kind
+ // conversion (e.g. 0x05 unmarshalling on string).
+ if outt.Elem().Kind() != reflect.Uint8 {
+ break
+ }
+ switch inv.Kind() {
+ case reflect.String:
+ slice := []byte(inv.String())
+ out.Set(reflect.ValueOf(slice))
+ return true
+ case reflect.Slice:
+ switch outt.Kind() {
+ case reflect.Array:
+ reflect.Copy(out, inv)
+ case reflect.Slice:
+ out.SetBytes(inv.Bytes())
+ }
+ return true
+ }
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ switch inv.Kind() {
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ out.SetInt(inv.Int())
+ return true
+ case reflect.Float32, reflect.Float64:
+ out.SetInt(int64(inv.Float()))
+ return true
+ case reflect.Bool:
+ if inv.Bool() {
+ out.SetInt(1)
+ } else {
+ out.SetInt(0)
+ }
+ return true
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ panic("can't happen: no uint types in BSON (!?)")
+ }
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ switch inv.Kind() {
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ out.SetUint(uint64(inv.Int()))
+ return true
+ case reflect.Float32, reflect.Float64:
+ out.SetUint(uint64(inv.Float()))
+ return true
+ case reflect.Bool:
+ if inv.Bool() {
+ out.SetUint(1)
+ } else {
+ out.SetUint(0)
+ }
+ return true
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ panic("Can't happen. No uint types in BSON.")
+ }
+ case reflect.Float32, reflect.Float64:
+ switch inv.Kind() {
+ case reflect.Float32, reflect.Float64:
+ out.SetFloat(inv.Float())
+ return true
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ out.SetFloat(float64(inv.Int()))
+ return true
+ case reflect.Bool:
+ if inv.Bool() {
+ out.SetFloat(1)
+ } else {
+ out.SetFloat(0)
+ }
+ return true
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ panic("Can't happen. No uint types in BSON?")
+ }
+ case reflect.Bool:
+ switch inv.Kind() {
+ case reflect.Bool:
+ out.SetBool(inv.Bool())
+ return true
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ out.SetBool(inv.Int() != 0)
+ return true
+ case reflect.Float32, reflect.Float64:
+ out.SetBool(inv.Float() != 0)
+ return true
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ panic("Can't happen. No uint types in BSON?")
+ }
+ case reflect.Struct:
+ if outt == typeURL && inv.Kind() == reflect.String {
+ u, err := url.Parse(inv.String())
+ if err != nil {
+ panic(err)
+ }
+ out.Set(reflect.ValueOf(u).Elem())
+ return true
+ }
+ if outt == typeBinary {
+ if b, ok := in.([]byte); ok {
+ out.Set(reflect.ValueOf(Binary{Data: b}))
+ return true
+ }
+ }
+ }
+
+ return false
+}
+
+// --------------------------------------------------------------------------
+// Parsers of basic types.
+
+func (d *decoder) readRegEx() RegEx {
+ re := RegEx{}
+ re.Pattern = d.readCStr()
+ re.Options = d.readCStr()
+ return re
+}
+
+func (d *decoder) readBinary() Binary {
+ l := d.readInt32()
+ b := Binary{}
+ b.Kind = d.readByte()
+ if b.Kind == BinaryBinaryOld && l > 4 {
+ // Weird obsolete format with redundant length.
+ rl := d.readInt32()
+ if rl != l-4 {
+ corrupted()
+ }
+ l = rl
+ }
+ b.Data = d.readBytes(l)
+ return b
+}
+
+func (d *decoder) readStr() string {
+ l := d.readInt32()
+ b := d.readBytes(l - 1)
+ if d.readByte() != '\x00' {
+ corrupted()
+ }
+ return string(b)
+}
+
+func (d *decoder) readCStr() string {
+ start := d.i
+ end := start
+ l := len(d.in)
+ for ; end != l; end++ {
+ if d.in[end] == '\x00' {
+ break
+ }
+ }
+ d.i = end + 1
+ if d.i > l {
+ corrupted()
+ }
+ return string(d.in[start:end])
+}
+
+func (d *decoder) readBool() bool {
+ b := d.readByte()
+ if b == 0 {
+ return false
+ }
+ if b == 1 {
+ return true
+ }
+ panic(fmt.Sprintf("encoded boolean must be 1 or 0, found %d", b))
+}
+
+func (d *decoder) readFloat64() float64 {
+ return math.Float64frombits(uint64(d.readInt64()))
+}
+
+func (d *decoder) readInt32() int32 {
+ b := d.readBytes(4)
+ return int32((uint32(b[0]) << 0) |
+ (uint32(b[1]) << 8) |
+ (uint32(b[2]) << 16) |
+ (uint32(b[3]) << 24))
+}
+
+func getSize(offset int, b []byte) (int, error) {
+ if offset+4 > len(b) {
+ return 0, io.ErrUnexpectedEOF
+ }
+ return int((uint32(b[offset]) << 0) |
+ (uint32(b[offset+1]) << 8) |
+ (uint32(b[offset+2]) << 16) |
+ (uint32(b[offset+3]) << 24)), nil
+}
+
+func (d *decoder) readInt64() int64 {
+ b := d.readBytes(8)
+ return int64((uint64(b[0]) << 0) |
+ (uint64(b[1]) << 8) |
+ (uint64(b[2]) << 16) |
+ (uint64(b[3]) << 24) |
+ (uint64(b[4]) << 32) |
+ (uint64(b[5]) << 40) |
+ (uint64(b[6]) << 48) |
+ (uint64(b[7]) << 56))
+}
+
+func (d *decoder) readByte() byte {
+ i := d.i
+ d.i++
+ if d.i > len(d.in) {
+ corrupted()
+ }
+ return d.in[i]
+}
+
+func (d *decoder) readBytes(length int32) []byte {
+ if length < 0 {
+ corrupted()
+ }
+ start := d.i
+ d.i += int(length)
+ if d.i < start || d.i > len(d.in) {
+ corrupted()
+ }
+ return d.in[start : start+int(length)]
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/encode.go b/backend/vendor/github.com/globalsign/mgo/bson/encode.go
new file mode 100644
index 00000000..d0c6b2a8
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/encode.go
@@ -0,0 +1,645 @@
+// BSON library for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// gobson - BSON library for Go.
+
+package bson
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "net/url"
+ "reflect"
+ "sort"
+ "strconv"
+ "sync"
+ "time"
+)
+
+// --------------------------------------------------------------------------
+// Some internal infrastructure.
+
+var (
+ typeBinary = reflect.TypeOf(Binary{})
+ typeObjectId = reflect.TypeOf(ObjectId(""))
+ typeDBPointer = reflect.TypeOf(DBPointer{"", ObjectId("")})
+ typeSymbol = reflect.TypeOf(Symbol(""))
+ typeMongoTimestamp = reflect.TypeOf(MongoTimestamp(0))
+ typeOrderKey = reflect.TypeOf(MinKey)
+ typeDocElem = reflect.TypeOf(DocElem{})
+ typeRawDocElem = reflect.TypeOf(RawDocElem{})
+ typeRaw = reflect.TypeOf(Raw{})
+ typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{}))
+ typeURL = reflect.TypeOf(url.URL{})
+ typeTime = reflect.TypeOf(time.Time{})
+ typeString = reflect.TypeOf("")
+ typeJSONNumber = reflect.TypeOf(json.Number(""))
+ typeTimeDuration = reflect.TypeOf(time.Duration(0))
+)
+
+var (
+ // spec for []uint8 or []byte encoding
+ arrayOps = map[string]bool{
+ "$in": true,
+ "$nin": true,
+ "$all": true,
+ }
+)
+
+const itoaCacheSize = 32
+
+const (
+ getterUnknown = iota
+ getterNone
+ getterTypeVal
+ getterTypePtr
+ getterAddr
+)
+
+var itoaCache []string
+
+var getterStyles map[reflect.Type]int
+var getterIface reflect.Type
+var getterMutex sync.RWMutex
+
+func init() {
+ itoaCache = make([]string, itoaCacheSize)
+ for i := 0; i != itoaCacheSize; i++ {
+ itoaCache[i] = strconv.Itoa(i)
+ }
+ var iface Getter
+ getterIface = reflect.TypeOf(&iface).Elem()
+ getterStyles = make(map[reflect.Type]int)
+}
+
+func itoa(i int) string {
+ if i < itoaCacheSize {
+ return itoaCache[i]
+ }
+ return strconv.Itoa(i)
+}
+
+func getterStyle(outt reflect.Type) int {
+ getterMutex.RLock()
+ style := getterStyles[outt]
+ getterMutex.RUnlock()
+ if style != getterUnknown {
+ return style
+ }
+
+ getterMutex.Lock()
+ defer getterMutex.Unlock()
+ if outt.Implements(getterIface) {
+ vt := outt
+ for vt.Kind() == reflect.Ptr {
+ vt = vt.Elem()
+ }
+ if vt.Implements(getterIface) {
+ style = getterTypeVal
+ } else {
+ style = getterTypePtr
+ }
+ } else if reflect.PtrTo(outt).Implements(getterIface) {
+ style = getterAddr
+ } else {
+ style = getterNone
+ }
+ getterStyles[outt] = style
+ return style
+}
+
+func getGetter(outt reflect.Type, out reflect.Value) Getter {
+ style := getterStyle(outt)
+ if style == getterNone {
+ return nil
+ }
+ if style == getterAddr {
+ if !out.CanAddr() {
+ return nil
+ }
+ return out.Addr().Interface().(Getter)
+ }
+ if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() {
+ return nil
+ }
+ return out.Interface().(Getter)
+}
+
+// --------------------------------------------------------------------------
+// Marshaling of the document value itself.
+
+type encoder struct {
+ out []byte
+}
+
+func (e *encoder) addDoc(v reflect.Value) {
+ for {
+ if vi, ok := v.Interface().(Getter); ok {
+ getv, err := vi.GetBSON()
+ if err != nil {
+ panic(err)
+ }
+ v = reflect.ValueOf(getv)
+ continue
+ }
+ if v.Kind() == reflect.Ptr {
+ v = v.Elem()
+ continue
+ }
+ break
+ }
+
+ if v.Type() == typeRaw {
+ raw := v.Interface().(Raw)
+ if raw.Kind != 0x03 && raw.Kind != 0x00 {
+ panic("Attempted to marshal Raw kind " + strconv.Itoa(int(raw.Kind)) + " as a document")
+ }
+ if len(raw.Data) == 0 {
+ panic("Attempted to marshal empty Raw document")
+ }
+ e.addBytes(raw.Data...)
+ return
+ }
+
+ start := e.reserveInt32()
+
+ switch v.Kind() {
+ case reflect.Map:
+ e.addMap(v)
+ case reflect.Struct:
+ e.addStruct(v)
+ case reflect.Array, reflect.Slice:
+ e.addSlice(v)
+ default:
+ panic("Can't marshal " + v.Type().String() + " as a BSON document")
+ }
+
+ e.addBytes(0)
+ e.setInt32(start, int32(len(e.out)-start))
+}
+
+func (e *encoder) addMap(v reflect.Value) {
+ for _, k := range v.MapKeys() {
+ e.addElem(fmt.Sprint(k), v.MapIndex(k), false)
+ }
+}
+
+func (e *encoder) addStruct(v reflect.Value) {
+ sinfo, err := getStructInfo(v.Type())
+ if err != nil {
+ panic(err)
+ }
+ var value reflect.Value
+ if sinfo.InlineMap >= 0 {
+ m := v.Field(sinfo.InlineMap)
+ if m.Len() > 0 {
+ for _, k := range m.MapKeys() {
+ ks := k.String()
+ if _, found := sinfo.FieldsMap[ks]; found {
+ panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", ks))
+ }
+ e.addElem(ks, m.MapIndex(k), false)
+ }
+ }
+ }
+ for _, info := range sinfo.FieldsList {
+ if info.Inline == nil {
+ value = v.Field(info.Num)
+ } else {
+ // as pointers to struct are allowed here,
+ // there is no guarantee that pointer won't be nil.
+ //
+ // It is expected allowed behaviour
+ // so info.Inline MAY consist index to a nil pointer
+ // and that is why we safely call v.FieldByIndex and just continue on panic
+ field, errField := safeFieldByIndex(v, info.Inline)
+ if errField != nil {
+ continue
+ }
+
+ value = field
+ }
+ if info.OmitEmpty && isZero(value) {
+ continue
+ }
+ if useRespectNilValues &&
+ (value.Kind() == reflect.Slice || value.Kind() == reflect.Map) &&
+ value.IsNil() {
+ e.addElem(info.Key, reflect.ValueOf(nil), info.MinSize)
+ continue
+ }
+ e.addElem(info.Key, value, info.MinSize)
+ }
+}
+
+func safeFieldByIndex(v reflect.Value, index []int) (result reflect.Value, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ switch r := recovered.(type) {
+ case string:
+ err = fmt.Errorf("%s", r)
+ case error:
+ err = r
+ }
+ }
+ }()
+
+ result = v.FieldByIndex(index)
+ return
+}
+
+func isZero(v reflect.Value) bool {
+ switch v.Kind() {
+ case reflect.String:
+ return len(v.String()) == 0
+ case reflect.Ptr, reflect.Interface:
+ return v.IsNil()
+ case reflect.Slice:
+ return v.Len() == 0
+ case reflect.Map:
+ return v.Len() == 0
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ return v.Int() == 0
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ return v.Uint() == 0
+ case reflect.Float32, reflect.Float64:
+ return v.Float() == 0
+ case reflect.Bool:
+ return !v.Bool()
+ case reflect.Struct:
+ vt := v.Type()
+ if vt == typeTime {
+ return v.Interface().(time.Time).IsZero()
+ }
+ for i := 0; i < v.NumField(); i++ {
+ if vt.Field(i).PkgPath != "" && !vt.Field(i).Anonymous {
+ continue // Private field
+ }
+ if !isZero(v.Field(i)) {
+ return false
+ }
+ }
+ return true
+ }
+ return false
+}
+
+func (e *encoder) addSlice(v reflect.Value) {
+ vi := v.Interface()
+ if d, ok := vi.(D); ok {
+ for _, elem := range d {
+ e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
+ }
+ return
+ }
+ if d, ok := vi.(RawD); ok {
+ for _, elem := range d {
+ e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
+ }
+ return
+ }
+ l := v.Len()
+ et := v.Type().Elem()
+ if et == typeDocElem {
+ for i := 0; i < l; i++ {
+ elem := v.Index(i).Interface().(DocElem)
+ e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
+ }
+ return
+ }
+ if et == typeRawDocElem {
+ for i := 0; i < l; i++ {
+ elem := v.Index(i).Interface().(RawDocElem)
+ e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
+ }
+ return
+ }
+ for i := 0; i < l; i++ {
+ e.addElem(itoa(i), v.Index(i), false)
+ }
+}
+
+// --------------------------------------------------------------------------
+// Marshaling of elements in a document.
+
+func (e *encoder) addElemName(kind byte, name string) {
+ e.addBytes(kind)
+ e.addBytes([]byte(name)...)
+ e.addBytes(0)
+}
+
+func (e *encoder) addElem(name string, v reflect.Value, minSize bool) {
+
+ if !v.IsValid() {
+ e.addElemName(0x0A, name)
+ return
+ }
+
+ if getter := getGetter(v.Type(), v); getter != nil {
+ getv, err := getter.GetBSON()
+ if err != nil {
+ panic(err)
+ }
+ e.addElem(name, reflect.ValueOf(getv), minSize)
+ return
+ }
+
+ switch v.Kind() {
+
+ case reflect.Interface:
+ e.addElem(name, v.Elem(), minSize)
+
+ case reflect.Ptr:
+ e.addElem(name, v.Elem(), minSize)
+
+ case reflect.String:
+ s := v.String()
+ switch v.Type() {
+ case typeObjectId:
+ if len(s) != 12 {
+ panic("ObjectIDs must be exactly 12 bytes long (got " +
+ strconv.Itoa(len(s)) + ")")
+ }
+ e.addElemName(0x07, name)
+ e.addBytes([]byte(s)...)
+ case typeSymbol:
+ e.addElemName(0x0E, name)
+ e.addStr(s)
+ case typeJSONNumber:
+ n := v.Interface().(json.Number)
+ if i, err := n.Int64(); err == nil {
+ e.addElemName(0x12, name)
+ e.addInt64(i)
+ } else if f, err := n.Float64(); err == nil {
+ e.addElemName(0x01, name)
+ e.addFloat64(f)
+ } else {
+ panic("failed to convert json.Number to a number: " + s)
+ }
+ default:
+ e.addElemName(0x02, name)
+ e.addStr(s)
+ }
+
+ case reflect.Float32, reflect.Float64:
+ e.addElemName(0x01, name)
+ e.addFloat64(v.Float())
+
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ u := v.Uint()
+ if int64(u) < 0 {
+ panic("BSON has no uint64 type, and value is too large to fit correctly in an int64")
+ } else if u <= math.MaxInt32 && (minSize || v.Kind() <= reflect.Uint32) {
+ e.addElemName(0x10, name)
+ e.addInt32(int32(u))
+ } else {
+ e.addElemName(0x12, name)
+ e.addInt64(int64(u))
+ }
+
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ switch v.Type() {
+ case typeMongoTimestamp:
+ e.addElemName(0x11, name)
+ e.addInt64(v.Int())
+
+ case typeOrderKey:
+ if v.Int() == int64(MaxKey) {
+ e.addElemName(0x7F, name)
+ } else {
+ e.addElemName(0xFF, name)
+ }
+ case typeTimeDuration:
+ // Stored as int64
+ e.addElemName(0x12, name)
+
+ e.addInt64(int64(v.Int() / 1e6))
+ default:
+ i := v.Int()
+ if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 {
+ // It fits into an int32, encode as such.
+ e.addElemName(0x10, name)
+ e.addInt32(int32(i))
+ } else {
+ e.addElemName(0x12, name)
+ e.addInt64(i)
+ }
+ }
+
+ case reflect.Bool:
+ e.addElemName(0x08, name)
+ if v.Bool() {
+ e.addBytes(1)
+ } else {
+ e.addBytes(0)
+ }
+
+ case reflect.Map:
+ e.addElemName(0x03, name)
+ e.addDoc(v)
+
+ case reflect.Slice:
+ vt := v.Type()
+ et := vt.Elem()
+ if et.Kind() == reflect.Uint8 {
+ if arrayOps[name] {
+ e.addElemName(0x04, name)
+ e.addDoc(v)
+ } else {
+ e.addElemName(0x05, name)
+ e.addBinary(0x00, v.Bytes())
+ }
+ } else if et == typeDocElem || et == typeRawDocElem {
+ e.addElemName(0x03, name)
+ e.addDoc(v)
+ } else {
+ e.addElemName(0x04, name)
+ e.addDoc(v)
+ }
+
+ case reflect.Array:
+ et := v.Type().Elem()
+ if et.Kind() == reflect.Uint8 {
+ if arrayOps[name] {
+ e.addElemName(0x04, name)
+ e.addDoc(v)
+ } else {
+ e.addElemName(0x05, name)
+ if v.CanAddr() {
+ e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte))
+ } else {
+ n := v.Len()
+ e.addInt32(int32(n))
+ e.addBytes(0x00)
+ for i := 0; i < n; i++ {
+ el := v.Index(i)
+ e.addBytes(byte(el.Uint()))
+ }
+ }
+ }
+ } else {
+ e.addElemName(0x04, name)
+ e.addDoc(v)
+ }
+
+ case reflect.Struct:
+ switch s := v.Interface().(type) {
+
+ case Raw:
+ kind := s.Kind
+ if kind == 0x00 {
+ kind = 0x03
+ }
+ if len(s.Data) == 0 && kind != 0x06 && kind != 0x0A && kind != 0xFF && kind != 0x7F {
+ panic("Attempted to marshal empty Raw document")
+ }
+ e.addElemName(kind, name)
+ e.addBytes(s.Data...)
+
+ case Binary:
+ e.addElemName(0x05, name)
+ e.addBinary(s.Kind, s.Data)
+
+ case Decimal128:
+ e.addElemName(0x13, name)
+ e.addInt64(int64(s.l))
+ e.addInt64(int64(s.h))
+
+ case DBPointer:
+ e.addElemName(0x0C, name)
+ e.addStr(s.Namespace)
+ if len(s.Id) != 12 {
+ panic("ObjectIDs must be exactly 12 bytes long (got " +
+ strconv.Itoa(len(s.Id)) + ")")
+ }
+ e.addBytes([]byte(s.Id)...)
+
+ case RegEx:
+ e.addElemName(0x0B, name)
+ e.addCStr(s.Pattern)
+ options := runes(s.Options)
+ sort.Sort(options)
+ e.addCStr(string(options))
+
+ case JavaScript:
+ if s.Scope == nil {
+ e.addElemName(0x0D, name)
+ e.addStr(s.Code)
+ } else {
+ e.addElemName(0x0F, name)
+ start := e.reserveInt32()
+ e.addStr(s.Code)
+ e.addDoc(reflect.ValueOf(s.Scope))
+ e.setInt32(start, int32(len(e.out)-start))
+ }
+
+ case time.Time:
+ // MongoDB handles timestamps as milliseconds.
+ e.addElemName(0x09, name)
+ e.addInt64(s.Unix()*1000 + int64(s.Nanosecond()/1e6))
+
+ case url.URL:
+ e.addElemName(0x02, name)
+ e.addStr(s.String())
+
+ case undefined:
+ e.addElemName(0x06, name)
+
+ default:
+ e.addElemName(0x03, name)
+ e.addDoc(v)
+ }
+
+ default:
+ panic("Can't marshal " + v.Type().String() + " in a BSON document")
+ }
+}
+
+// -------------
+// Helper method for sorting regex options
+type runes []rune
+
+func (a runes) Len() int { return len(a) }
+func (a runes) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a runes) Less(i, j int) bool { return a[i] < a[j] }
+
+// --------------------------------------------------------------------------
+// Marshaling of base types.
+
+func (e *encoder) addBinary(subtype byte, v []byte) {
+ if subtype == 0x02 {
+ // Wonder how that brilliant idea came to life. Obsolete, luckily.
+ e.addInt32(int32(len(v) + 4))
+ e.addBytes(subtype)
+ e.addInt32(int32(len(v)))
+ } else {
+ e.addInt32(int32(len(v)))
+ e.addBytes(subtype)
+ }
+ e.addBytes(v...)
+}
+
+func (e *encoder) addStr(v string) {
+ e.addInt32(int32(len(v) + 1))
+ e.addCStr(v)
+}
+
+func (e *encoder) addCStr(v string) {
+ e.addBytes([]byte(v)...)
+ e.addBytes(0)
+}
+
+func (e *encoder) reserveInt32() (pos int) {
+ pos = len(e.out)
+ e.addBytes(0, 0, 0, 0)
+ return pos
+}
+
+func (e *encoder) setInt32(pos int, v int32) {
+ e.out[pos+0] = byte(v)
+ e.out[pos+1] = byte(v >> 8)
+ e.out[pos+2] = byte(v >> 16)
+ e.out[pos+3] = byte(v >> 24)
+}
+
+func (e *encoder) addInt32(v int32) {
+ u := uint32(v)
+ e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24))
+}
+
+func (e *encoder) addInt64(v int64) {
+ u := uint64(v)
+ e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24),
+ byte(u>>32), byte(u>>40), byte(u>>48), byte(u>>56))
+}
+
+func (e *encoder) addFloat64(v float64) {
+ e.addInt64(int64(math.Float64bits(v)))
+}
+
+func (e *encoder) addBytes(v ...byte) {
+ e.out = append(e.out, v...)
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/json.go b/backend/vendor/github.com/globalsign/mgo/bson/json.go
new file mode 100644
index 00000000..045c7130
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/json.go
@@ -0,0 +1,384 @@
+package bson
+
+import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/globalsign/mgo/internal/json"
+)
+
+// UnmarshalJSON unmarshals a JSON value that may hold non-standard
+// syntax as defined in BSON's extended JSON specification.
+func UnmarshalJSON(data []byte, value interface{}) error {
+ d := json.NewDecoder(bytes.NewBuffer(data))
+ d.Extend(&jsonExt)
+ return d.Decode(value)
+}
+
+// MarshalJSON marshals a JSON value that may hold non-standard
+// syntax as defined in BSON's extended JSON specification.
+func MarshalJSON(value interface{}) ([]byte, error) {
+ var buf bytes.Buffer
+ e := json.NewEncoder(&buf)
+ e.Extend(&jsonExt)
+ err := e.Encode(value)
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
+
+// jdec is used internally by the JSON decoding functions
+// so they may unmarshal functions without getting into endless
+// recursion due to keyed objects.
+func jdec(data []byte, value interface{}) error {
+ d := json.NewDecoder(bytes.NewBuffer(data))
+ d.Extend(&funcExt)
+ return d.Decode(value)
+}
+
+var jsonExt json.Extension
+var funcExt json.Extension
+
+// TODO
+// - Shell regular expressions ("/regexp/opts")
+
+func init() {
+ jsonExt.DecodeUnquotedKeys(true)
+ jsonExt.DecodeTrailingCommas(true)
+
+ funcExt.DecodeFunc("BinData", "$binaryFunc", "$type", "$binary")
+ jsonExt.DecodeKeyed("$binary", jdecBinary)
+ jsonExt.DecodeKeyed("$binaryFunc", jdecBinary)
+ jsonExt.EncodeType([]byte(nil), jencBinarySlice)
+ jsonExt.EncodeType(Binary{}, jencBinaryType)
+
+ funcExt.DecodeFunc("ISODate", "$dateFunc", "S")
+ funcExt.DecodeFunc("new Date", "$dateFunc", "S")
+ jsonExt.DecodeKeyed("$date", jdecDate)
+ jsonExt.DecodeKeyed("$dateFunc", jdecDate)
+ jsonExt.EncodeType(time.Time{}, jencDate)
+
+ funcExt.DecodeFunc("Timestamp", "$timestamp", "t", "i")
+ jsonExt.DecodeKeyed("$timestamp", jdecTimestamp)
+ jsonExt.EncodeType(MongoTimestamp(0), jencTimestamp)
+
+ funcExt.DecodeConst("undefined", Undefined)
+
+ jsonExt.DecodeKeyed("$regex", jdecRegEx)
+ jsonExt.EncodeType(RegEx{}, jencRegEx)
+
+ funcExt.DecodeFunc("ObjectId", "$oidFunc", "Id")
+ jsonExt.DecodeKeyed("$oid", jdecObjectId)
+ jsonExt.DecodeKeyed("$oidFunc", jdecObjectId)
+ jsonExt.EncodeType(ObjectId(""), jencObjectId)
+
+ funcExt.DecodeFunc("DBRef", "$dbrefFunc", "$ref", "$id")
+ jsonExt.DecodeKeyed("$dbrefFunc", jdecDBRef)
+
+ funcExt.DecodeFunc("NumberLong", "$numberLongFunc", "N")
+ jsonExt.DecodeKeyed("$numberLong", jdecNumberLong)
+ jsonExt.DecodeKeyed("$numberLongFunc", jdecNumberLong)
+ jsonExt.EncodeType(int64(0), jencNumberLong)
+ jsonExt.EncodeType(int(0), jencInt)
+
+ funcExt.DecodeConst("MinKey", MinKey)
+ funcExt.DecodeConst("MaxKey", MaxKey)
+ jsonExt.DecodeKeyed("$minKey", jdecMinKey)
+ jsonExt.DecodeKeyed("$maxKey", jdecMaxKey)
+ jsonExt.EncodeType(orderKey(0), jencMinMaxKey)
+
+ jsonExt.DecodeKeyed("$undefined", jdecUndefined)
+ jsonExt.EncodeType(Undefined, jencUndefined)
+
+ jsonExt.Extend(&funcExt)
+}
+
+func fbytes(format string, args ...interface{}) []byte {
+ var buf bytes.Buffer
+ fmt.Fprintf(&buf, format, args...)
+ return buf.Bytes()
+}
+
+func jdecBinary(data []byte) (interface{}, error) {
+ var v struct {
+ Binary []byte `json:"$binary"`
+ Type string `json:"$type"`
+ Func struct {
+ Binary []byte `json:"$binary"`
+ Type int64 `json:"$type"`
+ } `json:"$binaryFunc"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+
+ var binData []byte
+ var binKind int64
+ if v.Type == "" && v.Binary == nil {
+ binData = v.Func.Binary
+ binKind = v.Func.Type
+ } else if v.Type == "" {
+ return v.Binary, nil
+ } else {
+ binData = v.Binary
+ binKind, err = strconv.ParseInt(v.Type, 0, 64)
+ if err != nil {
+ binKind = -1
+ }
+ }
+
+ if binKind == 0 {
+ return binData, nil
+ }
+ if binKind < 0 || binKind > 255 {
+ return nil, fmt.Errorf("invalid type in binary object: %s", data)
+ }
+
+ return Binary{Kind: byte(binKind), Data: binData}, nil
+}
+
+func jencBinarySlice(v interface{}) ([]byte, error) {
+ in := v.([]byte)
+ out := make([]byte, base64.StdEncoding.EncodedLen(len(in)))
+ base64.StdEncoding.Encode(out, in)
+ return fbytes(`{"$binary":"%s","$type":"0x0"}`, out), nil
+}
+
+func jencBinaryType(v interface{}) ([]byte, error) {
+ in := v.(Binary)
+ out := make([]byte, base64.StdEncoding.EncodedLen(len(in.Data)))
+ base64.StdEncoding.Encode(out, in.Data)
+ return fbytes(`{"$binary":"%s","$type":"0x%x"}`, out, in.Kind), nil
+}
+
+const jdateFormat = "2006-01-02T15:04:05.999Z07:00"
+
+func jdecDate(data []byte) (interface{}, error) {
+ var v struct {
+ S string `json:"$date"`
+ Func struct {
+ S string
+ } `json:"$dateFunc"`
+ }
+ _ = jdec(data, &v)
+ if v.S == "" {
+ v.S = v.Func.S
+ }
+ if v.S != "" {
+ var errs []string
+ for _, format := range []string{jdateFormat, "2006-01-02"} {
+ t, err := time.Parse(format, v.S)
+ if err == nil {
+ return t, nil
+ }
+ errs = append(errs, err.Error())
+ }
+ return nil, fmt.Errorf("cannot parse date: %q [%s]", v.S, strings.Join(errs, ", "))
+ }
+
+ var vn struct {
+ Date struct {
+ N int64 `json:"$numberLong,string"`
+ } `json:"$date"`
+ Func struct {
+ S int64
+ } `json:"$dateFunc"`
+ }
+ err := jdec(data, &vn)
+ if err != nil {
+ return nil, fmt.Errorf("cannot parse date: %q", data)
+ }
+ n := vn.Date.N
+ if n == 0 {
+ n = vn.Func.S
+ }
+ return time.Unix(n/1000, n%1000*1e6).UTC(), nil
+}
+
+func jencDate(v interface{}) ([]byte, error) {
+ t := v.(time.Time)
+ return fbytes(`{"$date":%q}`, t.Format(jdateFormat)), nil
+}
+
+func jdecTimestamp(data []byte) (interface{}, error) {
+ var v struct {
+ Func struct {
+ T int32 `json:"t"`
+ I int32 `json:"i"`
+ } `json:"$timestamp"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ return MongoTimestamp(uint64(v.Func.T)<<32 | uint64(uint32(v.Func.I))), nil
+}
+
+func jencTimestamp(v interface{}) ([]byte, error) {
+ ts := uint64(v.(MongoTimestamp))
+ return fbytes(`{"$timestamp":{"t":%d,"i":%d}}`, ts>>32, uint32(ts)), nil
+}
+
+func jdecRegEx(data []byte) (interface{}, error) {
+ var v struct {
+ Regex string `json:"$regex"`
+ Options string `json:"$options"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ return RegEx{v.Regex, v.Options}, nil
+}
+
+func jencRegEx(v interface{}) ([]byte, error) {
+ re := v.(RegEx)
+ type regex struct {
+ Regex string `json:"$regex"`
+ Options string `json:"$options"`
+ }
+ return json.Marshal(regex{re.Pattern, re.Options})
+}
+
+func jdecObjectId(data []byte) (interface{}, error) {
+ var v struct {
+ Id string `json:"$oid"`
+ Func struct {
+ Id string
+ } `json:"$oidFunc"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ if v.Id == "" {
+ v.Id = v.Func.Id
+ }
+ return ObjectIdHex(v.Id), nil
+}
+
+func jencObjectId(v interface{}) ([]byte, error) {
+ return fbytes(`{"$oid":"%s"}`, v.(ObjectId).Hex()), nil
+}
+
+func jdecDBRef(data []byte) (interface{}, error) {
+ // TODO Support unmarshaling $ref and $id into the input value.
+ var v struct {
+ Obj map[string]interface{} `json:"$dbrefFunc"`
+ }
+ // TODO Fix this. Must not be required.
+ v.Obj = make(map[string]interface{})
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ return v.Obj, nil
+}
+
+func jdecNumberLong(data []byte) (interface{}, error) {
+ var v struct {
+ N int64 `json:"$numberLong,string"`
+ Func struct {
+ N int64 `json:",string"`
+ } `json:"$numberLongFunc"`
+ }
+ var vn struct {
+ N int64 `json:"$numberLong"`
+ Func struct {
+ N int64
+ } `json:"$numberLongFunc"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ err = jdec(data, &vn)
+ v.N = vn.N
+ v.Func.N = vn.Func.N
+ }
+ if err != nil {
+ return nil, err
+ }
+ if v.N != 0 {
+ return v.N, nil
+ }
+ return v.Func.N, nil
+}
+
+func jencNumberLong(v interface{}) ([]byte, error) {
+ n := v.(int64)
+ f := `{"$numberLong":"%d"}`
+ if n <= 1<<53 {
+ f = `{"$numberLong":%d}`
+ }
+ return fbytes(f, n), nil
+}
+
+func jencInt(v interface{}) ([]byte, error) {
+ n := v.(int)
+ f := `{"$numberLong":"%d"}`
+ if int64(n) <= 1<<53 {
+ f = `%d`
+ }
+ return fbytes(f, n), nil
+}
+
+func jdecMinKey(data []byte) (interface{}, error) {
+ var v struct {
+ N int64 `json:"$minKey"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ if v.N != 1 {
+ return nil, fmt.Errorf("invalid $minKey object: %s", data)
+ }
+ return MinKey, nil
+}
+
+func jdecMaxKey(data []byte) (interface{}, error) {
+ var v struct {
+ N int64 `json:"$maxKey"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ if v.N != 1 {
+ return nil, fmt.Errorf("invalid $maxKey object: %s", data)
+ }
+ return MaxKey, nil
+}
+
+func jencMinMaxKey(v interface{}) ([]byte, error) {
+ switch v.(orderKey) {
+ case MinKey:
+ return []byte(`{"$minKey":1}`), nil
+ case MaxKey:
+ return []byte(`{"$maxKey":1}`), nil
+ }
+ panic(fmt.Sprintf("invalid $minKey/$maxKey value: %d", v))
+}
+
+func jdecUndefined(data []byte) (interface{}, error) {
+ var v struct {
+ B bool `json:"$undefined"`
+ }
+ err := jdec(data, &v)
+ if err != nil {
+ return nil, err
+ }
+ if !v.B {
+ return nil, fmt.Errorf("invalid $undefined object: %s", data)
+ }
+ return Undefined, nil
+}
+
+func jencUndefined(v interface{}) ([]byte, error) {
+ return []byte(`{"$undefined":true}`), nil
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bson/stream.go b/backend/vendor/github.com/globalsign/mgo/bson/stream.go
new file mode 100644
index 00000000..46652845
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bson/stream.go
@@ -0,0 +1,90 @@
+package bson
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "io"
+)
+
+const (
+ // MinDocumentSize is the size of the smallest possible valid BSON document:
+ // an int32 size header + 0x00 (end of document).
+ MinDocumentSize = 5
+
+ // MaxDocumentSize is the largest possible size for a BSON document allowed by MongoDB,
+ // that is, 16 MiB (see https://docs.mongodb.com/manual/reference/limits/).
+ MaxDocumentSize = 16777216
+)
+
+// ErrInvalidDocumentSize is an error returned when a BSON document's header
+// contains a size smaller than MinDocumentSize or greater than MaxDocumentSize.
+type ErrInvalidDocumentSize struct {
+ DocumentSize int32
+}
+
+func (e ErrInvalidDocumentSize) Error() string {
+ return fmt.Sprintf("invalid document size %d", e.DocumentSize)
+}
+
+// A Decoder reads and decodes BSON values from an input stream.
+type Decoder struct {
+ source io.Reader
+}
+
+// NewDecoder returns a new Decoder that reads from source.
+// It does not add any extra buffering, and may not read data from source beyond the BSON values requested.
+func NewDecoder(source io.Reader) *Decoder {
+ return &Decoder{source: source}
+}
+
+// Decode reads the next BSON-encoded value from its input and stores it in the value pointed to by v.
+// See the documentation for Unmarshal for details about the conversion of BSON into a Go value.
+func (dec *Decoder) Decode(v interface{}) (err error) {
+ // BSON documents start with their size as a *signed* int32.
+ var docSize int32
+ if err = binary.Read(dec.source, binary.LittleEndian, &docSize); err != nil {
+ return
+ }
+
+ if docSize < MinDocumentSize || docSize > MaxDocumentSize {
+ return ErrInvalidDocumentSize{DocumentSize: docSize}
+ }
+
+ docBuffer := bytes.NewBuffer(make([]byte, 0, docSize))
+ if err = binary.Write(docBuffer, binary.LittleEndian, docSize); err != nil {
+ return
+ }
+
+ // docSize is the *full* document's size (including the 4-byte size header,
+ // which has already been read).
+ if _, err = io.CopyN(docBuffer, dec.source, int64(docSize-4)); err != nil {
+ return
+ }
+
+ // Let Unmarshal handle the rest.
+ defer handleErr(&err)
+ return Unmarshal(docBuffer.Bytes(), v)
+}
+
+// An Encoder encodes and writes BSON values to an output stream.
+type Encoder struct {
+ target io.Writer
+}
+
+// NewEncoder returns a new Encoder that writes to target.
+func NewEncoder(target io.Writer) *Encoder {
+ return &Encoder{target: target}
+}
+
+// Encode encodes v to BSON, and if successful writes it to the Encoder's output stream.
+// See the documentation for Marshal for details about the conversion of Go values to BSON.
+func (enc *Encoder) Encode(v interface{}) error {
+ data, err := Marshal(v)
+ if err != nil {
+ return err
+ }
+
+ _, err = enc.target.Write(data)
+ return err
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/bulk.go b/backend/vendor/github.com/globalsign/mgo/bulk.go
new file mode 100644
index 00000000..c234fcce
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/bulk.go
@@ -0,0 +1,366 @@
+package mgo
+
+import (
+ "bytes"
+ "sort"
+ "sync"
+
+ "github.com/globalsign/mgo/bson"
+)
+
+// Bulk represents an operation that can be prepared with several
+// orthogonal changes before being delivered to the server.
+//
+// MongoDB servers older than version 2.6 do not have proper support for bulk
+// operations, so the driver attempts to map its API as much as possible into
+// the functionality that works. In particular, in those releases updates and
+// removals are sent individually, and inserts are sent in bulk but have
+// suboptimal error reporting compared to more recent versions of the server.
+// See the documentation of BulkErrorCase for details on that.
+//
+// Relevant documentation:
+//
+// http://blog.mongodb.org/post/84922794768/mongodbs-new-bulk-api
+//
+type Bulk struct {
+ c *Collection
+ opcount int
+ actions []bulkAction
+ ordered bool
+}
+
+type bulkOp int
+
+const (
+ bulkInsert bulkOp = iota + 1
+ bulkUpdate
+ bulkUpdateAll
+ bulkRemove
+)
+
+type bulkAction struct {
+ op bulkOp
+ docs []interface{}
+ idxs []int
+}
+
+type bulkUpdateOp []interface{}
+type bulkDeleteOp []interface{}
+
+// BulkResult holds the results for a bulk operation.
+type BulkResult struct {
+ Matched int
+ Modified int // Available only for MongoDB 2.6+
+
+ // Be conservative while we understand exactly how to report these
+ // results in a useful and convenient way, and also how to emulate
+ // them with prior servers.
+ private bool
+}
+
+// BulkError holds an error returned from running a Bulk operation.
+// Individual errors may be obtained and inspected via the Cases method.
+type BulkError struct {
+ ecases []BulkErrorCase
+}
+
+func (e *BulkError) Error() string {
+ if len(e.ecases) == 0 {
+ return "invalid BulkError instance: no errors"
+ }
+ if len(e.ecases) == 1 {
+ return e.ecases[0].Err.Error()
+ }
+ msgs := make([]string, 0, len(e.ecases))
+ seen := make(map[string]bool)
+ for _, ecase := range e.ecases {
+ msg := ecase.Err.Error()
+ if !seen[msg] {
+ seen[msg] = true
+ msgs = append(msgs, msg)
+ }
+ }
+ if len(msgs) == 1 {
+ return msgs[0]
+ }
+ var buf bytes.Buffer
+ buf.WriteString("multiple errors in bulk operation:\n")
+ for _, msg := range msgs {
+ buf.WriteString(" - ")
+ buf.WriteString(msg)
+ buf.WriteByte('\n')
+ }
+ return buf.String()
+}
+
+type bulkErrorCases []BulkErrorCase
+
+func (slice bulkErrorCases) Len() int { return len(slice) }
+func (slice bulkErrorCases) Less(i, j int) bool { return slice[i].Index < slice[j].Index }
+func (slice bulkErrorCases) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
+
+// BulkErrorCase holds an individual error found while attempting a single change
+// within a bulk operation, and the position in which it was enqueued.
+//
+// MongoDB servers older than version 2.6 do not have proper support for bulk
+// operations, so the driver attempts to map its API as much as possible into
+// the functionality that works. In particular, only the last error is reported
+// for bulk inserts and without any positional information, so the Index
+// field is set to -1 in these cases.
+type BulkErrorCase struct {
+ Index int // Position of operation that failed, or -1 if unknown.
+ Err error
+}
+
+// Cases returns all individual errors found while attempting the requested changes.
+//
+// See the documentation of BulkErrorCase for limitations in older MongoDB releases.
+func (e *BulkError) Cases() []BulkErrorCase {
+ return e.ecases
+}
+
+var actionPool = sync.Pool{
+ New: func() interface{} {
+ return &bulkAction{
+ docs: make([]interface{}, 0),
+ idxs: make([]int, 0),
+ }
+ },
+}
+
+// Bulk returns a value to prepare the execution of a bulk operation.
+func (c *Collection) Bulk() *Bulk {
+ return &Bulk{c: c, ordered: true}
+}
+
+// Unordered puts the bulk operation in unordered mode.
+//
+// In unordered mode the indvidual operations may be sent
+// out of order, which means latter operations may proceed
+// even if prior ones have failed.
+func (b *Bulk) Unordered() {
+ b.ordered = false
+}
+
+func (b *Bulk) action(op bulkOp, opcount int) *bulkAction {
+ var action *bulkAction
+ if len(b.actions) > 0 && b.actions[len(b.actions)-1].op == op {
+ action = &b.actions[len(b.actions)-1]
+ } else if !b.ordered {
+ for i := range b.actions {
+ if b.actions[i].op == op {
+ action = &b.actions[i]
+ break
+ }
+ }
+ }
+ if action == nil {
+ a := actionPool.Get().(*bulkAction)
+ a.op = op
+ b.actions = append(b.actions, *a)
+ action = &b.actions[len(b.actions)-1]
+ }
+ for i := 0; i < opcount; i++ {
+ action.idxs = append(action.idxs, b.opcount)
+ b.opcount++
+ }
+ return action
+}
+
+// Insert queues up the provided documents for insertion.
+func (b *Bulk) Insert(docs ...interface{}) {
+ action := b.action(bulkInsert, len(docs))
+ action.docs = append(action.docs, docs...)
+}
+
+// Remove queues up the provided selectors for removing matching documents.
+// Each selector will remove only a single matching document.
+func (b *Bulk) Remove(selectors ...interface{}) {
+ action := b.action(bulkRemove, len(selectors))
+ for _, selector := range selectors {
+ if selector == nil {
+ selector = bson.D{}
+ }
+ action.docs = append(action.docs, &deleteOp{
+ Collection: b.c.FullName,
+ Selector: selector,
+ Flags: 1,
+ Limit: 1,
+ })
+ }
+}
+
+// RemoveAll queues up the provided selectors for removing all matching documents.
+// Each selector will remove all matching documents.
+func (b *Bulk) RemoveAll(selectors ...interface{}) {
+ action := b.action(bulkRemove, len(selectors))
+ for _, selector := range selectors {
+ if selector == nil {
+ selector = bson.D{}
+ }
+ action.docs = append(action.docs, &deleteOp{
+ Collection: b.c.FullName,
+ Selector: selector,
+ Flags: 0,
+ Limit: 0,
+ })
+ }
+}
+
+// Update queues up the provided pairs of updating instructions.
+// The first element of each pair selects which documents must be
+// updated, and the second element defines how to update it.
+// Each pair matches exactly one document for updating at most.
+func (b *Bulk) Update(pairs ...interface{}) {
+ if len(pairs)%2 != 0 {
+ panic("Bulk.Update requires an even number of parameters")
+ }
+ action := b.action(bulkUpdate, len(pairs)/2)
+ for i := 0; i < len(pairs); i += 2 {
+ selector := pairs[i]
+ if selector == nil {
+ selector = bson.D{}
+ }
+ action.docs = append(action.docs, &updateOp{
+ Collection: b.c.FullName,
+ Selector: selector,
+ Update: pairs[i+1],
+ })
+ }
+}
+
+// UpdateAll queues up the provided pairs of updating instructions.
+// The first element of each pair selects which documents must be
+// updated, and the second element defines how to update it.
+// Each pair updates all documents matching the selector.
+func (b *Bulk) UpdateAll(pairs ...interface{}) {
+ if len(pairs)%2 != 0 {
+ panic("Bulk.UpdateAll requires an even number of parameters")
+ }
+ action := b.action(bulkUpdate, len(pairs)/2)
+ for i := 0; i < len(pairs); i += 2 {
+ selector := pairs[i]
+ if selector == nil {
+ selector = bson.D{}
+ }
+ action.docs = append(action.docs, &updateOp{
+ Collection: b.c.FullName,
+ Selector: selector,
+ Update: pairs[i+1],
+ Flags: 2,
+ Multi: true,
+ })
+ }
+}
+
+// Upsert queues up the provided pairs of upserting instructions.
+// The first element of each pair selects which documents must be
+// updated, and the second element defines how to update it.
+// Each pair matches exactly one document for updating at most.
+func (b *Bulk) Upsert(pairs ...interface{}) {
+ if len(pairs)%2 != 0 {
+ panic("Bulk.Update requires an even number of parameters")
+ }
+ action := b.action(bulkUpdate, len(pairs)/2)
+ for i := 0; i < len(pairs); i += 2 {
+ selector := pairs[i]
+ if selector == nil {
+ selector = bson.D{}
+ }
+ action.docs = append(action.docs, &updateOp{
+ Collection: b.c.FullName,
+ Selector: selector,
+ Update: pairs[i+1],
+ Flags: 1,
+ Upsert: true,
+ })
+ }
+}
+
+// Run runs all the operations queued up.
+//
+// If an error is reported on an unordered bulk operation, the error value may
+// be an aggregation of all issues observed. As an exception to that, Insert
+// operations running on MongoDB versions prior to 2.6 will report the last
+// error only due to a limitation in the wire protocol.
+func (b *Bulk) Run() (*BulkResult, error) {
+ var result BulkResult
+ var berr BulkError
+ var failed bool
+ for i := range b.actions {
+ action := &b.actions[i]
+ var ok bool
+ switch action.op {
+ case bulkInsert:
+ ok = b.runInsert(action, &result, &berr)
+ case bulkUpdate:
+ ok = b.runUpdate(action, &result, &berr)
+ case bulkRemove:
+ ok = b.runRemove(action, &result, &berr)
+ default:
+ panic("unknown bulk operation")
+ }
+ action.idxs = action.idxs[0:0]
+ action.docs = action.docs[0:0]
+ actionPool.Put(action)
+ if !ok {
+ failed = true
+ if b.ordered {
+ break
+ }
+ }
+ }
+ if failed {
+ sort.Sort(bulkErrorCases(berr.ecases))
+ return nil, &berr
+ }
+ return &result, nil
+}
+
+func (b *Bulk) runInsert(action *bulkAction, result *BulkResult, berr *BulkError) bool {
+ op := &insertOp{b.c.FullName, action.docs, 0}
+ if !b.ordered {
+ op.flags = 1 // ContinueOnError
+ }
+ lerr, err := b.c.writeOp(op, b.ordered)
+ return b.checkSuccess(action, berr, lerr, err)
+}
+
+func (b *Bulk) runUpdate(action *bulkAction, result *BulkResult, berr *BulkError) bool {
+ lerr, err := b.c.writeOp(bulkUpdateOp(action.docs), b.ordered)
+ if lerr != nil {
+ result.Matched += lerr.N
+ result.Modified += lerr.modified
+ }
+ return b.checkSuccess(action, berr, lerr, err)
+}
+
+func (b *Bulk) runRemove(action *bulkAction, result *BulkResult, berr *BulkError) bool {
+ lerr, err := b.c.writeOp(bulkDeleteOp(action.docs), b.ordered)
+ if lerr != nil {
+ result.Matched += lerr.N
+ result.Modified += lerr.modified
+ }
+ return b.checkSuccess(action, berr, lerr, err)
+}
+
+func (b *Bulk) checkSuccess(action *bulkAction, berr *BulkError, lerr *LastError, err error) bool {
+ if lerr != nil && len(lerr.ecases) > 0 {
+ for i := 0; i < len(lerr.ecases); i++ {
+ // Map back from the local error index into the visible one.
+ ecase := lerr.ecases[i]
+ idx := ecase.Index
+ if idx >= 0 {
+ idx = action.idxs[idx]
+ }
+ berr.ecases = append(berr.ecases, BulkErrorCase{idx, ecase.Err})
+ }
+ return false
+ } else if err != nil {
+ for i := 0; i < len(action.idxs); i++ {
+ berr.ecases = append(berr.ecases, BulkErrorCase{action.idxs[i], err})
+ }
+ return false
+ }
+ return true
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/changestreams.go b/backend/vendor/github.com/globalsign/mgo/changestreams.go
new file mode 100644
index 00000000..5c2279c6
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/changestreams.go
@@ -0,0 +1,357 @@
+package mgo
+
+import (
+ "errors"
+ "fmt"
+ "reflect"
+ "sync"
+ "time"
+
+ "github.com/globalsign/mgo/bson"
+)
+
+type FullDocument string
+
+const (
+ Default = "default"
+ UpdateLookup = "updateLookup"
+)
+
+type ChangeStream struct {
+ iter *Iter
+ isClosed bool
+ options ChangeStreamOptions
+ pipeline interface{}
+ resumeToken *bson.Raw
+ collection *Collection
+ readPreference *ReadPreference
+ err error
+ m sync.Mutex
+ sessionCopied bool
+}
+
+type ChangeStreamOptions struct {
+
+ // FullDocument controls the amount of data that the server will return when
+ // returning a changes document.
+ FullDocument FullDocument
+
+ // ResumeAfter specifies the logical starting point for the new change stream.
+ ResumeAfter *bson.Raw
+
+ // MaxAwaitTimeMS specifies the maximum amount of time for the server to wait
+ // on new documents to satisfy a change stream query.
+ MaxAwaitTimeMS time.Duration
+
+ // BatchSize specifies the number of documents to return per batch.
+ BatchSize int
+
+ // Collation specifies the way the server should collate returned data.
+ //TODO Collation *Collation
+}
+
+var errMissingResumeToken = errors.New("resume token missing from result")
+
+// Watch constructs a new ChangeStream capable of receiving continuing data
+// from the database.
+func (coll *Collection) Watch(pipeline interface{},
+ options ChangeStreamOptions) (*ChangeStream, error) {
+
+ if pipeline == nil {
+ pipeline = []bson.M{}
+ }
+
+ csPipe := constructChangeStreamPipeline(pipeline, options)
+ pipe := coll.Pipe(&csPipe)
+ if options.MaxAwaitTimeMS > 0 {
+ pipe.SetMaxTime(options.MaxAwaitTimeMS)
+ }
+ if options.BatchSize > 0 {
+ pipe.Batch(options.BatchSize)
+ }
+ pIter := pipe.Iter()
+
+ // check that there was no issue creating the iterator.
+ // this will fail immediately with an error from the server if running against
+ // a standalone.
+ if err := pIter.Err(); err != nil {
+ return nil, err
+ }
+
+ pIter.isChangeStream = true
+ return &ChangeStream{
+ iter: pIter,
+ collection: coll,
+ resumeToken: nil,
+ options: options,
+ pipeline: pipeline,
+ }, nil
+}
+
+// Next retrieves the next document from the change stream, blocking if necessary.
+// Next returns true if a document was successfully unmarshalled into result,
+// and false if an error occured. When Next returns false, the Err method should
+// be called to check what error occurred during iteration. If there were no events
+// available (ErrNotFound), the Err method returns nil so the user can retry the invocaton.
+//
+// For example:
+//
+// pipeline := []bson.M{}
+//
+// changeStream := collection.Watch(pipeline, ChangeStreamOptions{})
+// for changeStream.Next(&changeDoc) {
+// fmt.Printf("Change: %v\n", changeDoc)
+// }
+//
+// if err := changeStream.Close(); err != nil {
+// return err
+// }
+//
+// If the pipeline used removes the _id field from the result, Next will error
+// because the _id field is needed to resume iteration when an error occurs.
+//
+func (changeStream *ChangeStream) Next(result interface{}) bool {
+ // the err field is being constantly overwritten and we don't want the user to
+ // attempt to read it at this point so we lock.
+ changeStream.m.Lock()
+
+ defer changeStream.m.Unlock()
+
+ // if we are in a state of error, then don't continue.
+ if changeStream.err != nil {
+ return false
+ }
+
+ if changeStream.isClosed {
+ changeStream.err = fmt.Errorf("illegal use of a closed ChangeStream")
+ return false
+ }
+
+ var err error
+
+ // attempt to fetch the change stream result.
+ err = changeStream.fetchResultSet(result)
+ if err == nil {
+ return true
+ }
+
+ // if we get no results we return false with no errors so the user can call Next
+ // again, resuming is not needed as the iterator is simply timed out as no events happened.
+ // The user will call Timeout in order to understand if this was the case.
+ if err == ErrNotFound {
+ return false
+ }
+
+ // check if the error is resumable
+ if !isResumableError(err) {
+ // error is not resumable, give up and return it to the user.
+ changeStream.err = err
+ return false
+ }
+
+ // try to resume.
+ err = changeStream.resume()
+ if err != nil {
+ // we've not been able to successfully resume and should only try once,
+ // so we give up.
+ changeStream.err = err
+ return false
+ }
+
+ // we've successfully resumed the changestream.
+ // try to fetch the next result.
+ err = changeStream.fetchResultSet(result)
+ if err != nil {
+ changeStream.err = err
+ return false
+ }
+
+ return true
+}
+
+// Err returns nil if no errors happened during iteration, or the actual
+// error otherwise.
+func (changeStream *ChangeStream) Err() error {
+ changeStream.m.Lock()
+ defer changeStream.m.Unlock()
+ return changeStream.err
+}
+
+// Close kills the server cursor used by the iterator, if any, and returns
+// nil if no errors happened during iteration, or the actual error otherwise.
+func (changeStream *ChangeStream) Close() error {
+ changeStream.m.Lock()
+ defer changeStream.m.Unlock()
+ changeStream.isClosed = true
+ err := changeStream.iter.Close()
+ if err != nil {
+ changeStream.err = err
+ }
+ if changeStream.sessionCopied {
+ changeStream.iter.session.Close()
+ changeStream.sessionCopied = false
+ }
+ return err
+}
+
+// ResumeToken returns a copy of the current resume token held by the change stream.
+// This token should be treated as an opaque token that can be provided to instantiate
+// a new change stream.
+func (changeStream *ChangeStream) ResumeToken() *bson.Raw {
+ changeStream.m.Lock()
+ defer changeStream.m.Unlock()
+ if changeStream.resumeToken == nil {
+ return nil
+ }
+ var tokenCopy = *changeStream.resumeToken
+ return &tokenCopy
+}
+
+// Timeout returns true if the last call of Next returned false because of an iterator timeout.
+func (changeStream *ChangeStream) Timeout() bool {
+ return changeStream.iter.Timeout()
+}
+
+func constructChangeStreamPipeline(pipeline interface{},
+ options ChangeStreamOptions) interface{} {
+ pipelinev := reflect.ValueOf(pipeline)
+
+ // ensure that the pipeline passed in is a slice.
+ if pipelinev.Kind() != reflect.Slice {
+ panic("pipeline argument must be a slice")
+ }
+
+ // construct the options to be used by the change notification
+ // pipeline stage.
+ changeStreamStageOptions := bson.M{}
+
+ if options.FullDocument != "" {
+ changeStreamStageOptions["fullDocument"] = options.FullDocument
+ }
+ if options.ResumeAfter != nil {
+ changeStreamStageOptions["resumeAfter"] = options.ResumeAfter
+ }
+
+ changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions}
+
+ pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1)
+
+ // insert the change notification pipeline stage at the beginning of the
+ // aggregation.
+ pipeOfInterfaces[0] = changeStreamStage
+
+ // convert the passed in slice to a slice of interfaces.
+ for i := 0; i < pipelinev.Len(); i++ {
+ pipeOfInterfaces[1+i] = pipelinev.Index(i).Addr().Interface()
+ }
+ var pipelineAsInterface interface{} = pipeOfInterfaces
+ return pipelineAsInterface
+}
+
+func (changeStream *ChangeStream) resume() error {
+ // copy the information for the new socket.
+
+ // Thanks to Copy() future uses will acquire a new socket against the newly selected DB.
+ newSession := changeStream.iter.session.Copy()
+
+ // fetch the cursor from the iterator and use it to run a killCursors
+ // on the connection.
+ cursorId := changeStream.iter.op.cursorId
+ err := runKillCursorsOnSession(newSession, cursorId)
+ if err != nil {
+ return err
+ }
+
+ // change out the old connection to the database with the new connection.
+ if changeStream.sessionCopied {
+ changeStream.collection.Database.Session.Close()
+ }
+ changeStream.collection.Database.Session = newSession
+ changeStream.sessionCopied = true
+
+ opts := changeStream.options
+ if changeStream.resumeToken != nil {
+ opts.ResumeAfter = changeStream.resumeToken
+ }
+ // make a new pipeline containing the resume token.
+ changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, opts)
+
+ // generate the new iterator with the new connection.
+ newPipe := changeStream.collection.Pipe(changeStreamPipeline)
+ changeStream.iter = newPipe.Iter()
+ if err := changeStream.iter.Err(); err != nil {
+ return err
+ }
+ changeStream.iter.isChangeStream = true
+ return nil
+}
+
+// fetchResumeToken unmarshals the _id field from the document, setting an error
+// on the changeStream if it is unable to.
+func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error {
+ changeStreamResult := struct {
+ ResumeToken *bson.Raw `bson:"_id,omitempty"`
+ }{}
+
+ err := rawResult.Unmarshal(&changeStreamResult)
+ if err != nil {
+ return err
+ }
+
+ if changeStreamResult.ResumeToken == nil {
+ return errMissingResumeToken
+ }
+
+ changeStream.resumeToken = changeStreamResult.ResumeToken
+ return nil
+}
+
+func (changeStream *ChangeStream) fetchResultSet(result interface{}) error {
+ rawResult := bson.Raw{}
+
+ // fetch the next set of documents from the cursor.
+ gotNext := changeStream.iter.Next(&rawResult)
+ err := changeStream.iter.Err()
+ if err != nil {
+ return err
+ }
+
+ if !gotNext && err == nil {
+ // If the iter.Err() method returns nil despite us not getting a next batch,
+ // it is becuase iter.Err() silences this case.
+ return ErrNotFound
+ }
+
+ // grab the resumeToken from the results
+ if err := changeStream.fetchResumeToken(&rawResult); err != nil {
+ return err
+ }
+
+ // put the raw results into the data structure the user provided.
+ if err := rawResult.Unmarshal(result); err != nil {
+ return err
+ }
+ return nil
+}
+
+func isResumableError(err error) bool {
+ _, isQueryError := err.(*QueryError)
+ // if it is not a database error OR it is a database error,
+ // but the error is a notMaster error
+ //and is not a missingResumeToken error (caused by the user provided pipeline)
+ return (!isQueryError || isNotMasterError(err)) && (err != errMissingResumeToken)
+}
+
+func runKillCursorsOnSession(session *Session, cursorId int64) error {
+ socket, err := session.acquireSocket(true)
+ if err != nil {
+ return err
+ }
+ err = socket.Query(&killCursorsOp{[]int64{cursorId}})
+ if err != nil {
+ return err
+ }
+ socket.Release()
+
+ return nil
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/cluster.go b/backend/vendor/github.com/globalsign/mgo/cluster.go
new file mode 100644
index 00000000..ff431cac
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/cluster.go
@@ -0,0 +1,704 @@
+// mgo - MongoDB driver for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package mgo
+
+import (
+ "errors"
+ "fmt"
+ "net"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/globalsign/mgo/bson"
+)
+
+// ---------------------------------------------------------------------------
+// Mongo cluster encapsulation.
+//
+// A cluster enables the communication with one or more servers participating
+// in a mongo cluster. This works with individual servers, a replica set,
+// a replica pair, one or multiple mongos routers, etc.
+
+type mongoCluster struct {
+ sync.RWMutex
+ serverSynced sync.Cond
+ userSeeds []string
+ dynaSeeds []string
+ servers mongoServers
+ masters mongoServers
+ references int
+ syncing bool
+ syncCount uint
+ cachedIndex map[string]bool
+ sync chan bool
+ dial dialer
+ dialInfo *DialInfo
+}
+
+func newCluster(userSeeds []string, info *DialInfo) *mongoCluster {
+ cluster := &mongoCluster{
+ userSeeds: userSeeds,
+ references: 1,
+ dial: dialer{info.Dial, info.DialServer},
+ dialInfo: info,
+ }
+ cluster.serverSynced.L = cluster.RWMutex.RLocker()
+ cluster.sync = make(chan bool, 1)
+ stats.cluster(+1)
+ go cluster.syncServersLoop()
+ return cluster
+}
+
+// Acquire increases the reference count for the cluster.
+func (cluster *mongoCluster) Acquire() {
+ cluster.Lock()
+ cluster.references++
+ debugf("Cluster %p acquired (refs=%d)", cluster, cluster.references)
+ cluster.Unlock()
+}
+
+// Release decreases the reference count for the cluster. Once
+// it reaches zero, all servers will be closed.
+func (cluster *mongoCluster) Release() {
+ cluster.Lock()
+ if cluster.references == 0 {
+ panic("cluster.Release() with references == 0")
+ }
+ cluster.references--
+ debugf("Cluster %p released (refs=%d)", cluster, cluster.references)
+ if cluster.references == 0 {
+ for _, server := range cluster.servers.Slice() {
+ server.Close()
+ }
+ // Wake up the sync loop so it can die.
+ cluster.syncServers()
+ stats.cluster(-1)
+ }
+ cluster.Unlock()
+}
+
+func (cluster *mongoCluster) LiveServers() (servers []string) {
+ cluster.RLock()
+ for _, serv := range cluster.servers.Slice() {
+ servers = append(servers, serv.Addr)
+ }
+ cluster.RUnlock()
+ return servers
+}
+
+func (cluster *mongoCluster) removeServer(server *mongoServer) {
+ cluster.Lock()
+ cluster.masters.Remove(server)
+ other := cluster.servers.Remove(server)
+ cluster.Unlock()
+ if other != nil {
+ other.CloseIdle()
+ log("Removed server ", server.Addr, " from cluster.")
+ }
+ server.CloseIdle()
+}
+
+type isMasterResult struct {
+ IsMaster bool
+ Secondary bool
+ Primary string
+ Hosts []string
+ Passives []string
+ Tags bson.D
+ Msg string
+ SetName string `bson:"setName"`
+ MaxWireVersion int `bson:"maxWireVersion"`
+}
+
+func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResult) error {
+ // Monotonic let's it talk to a slave and still hold the socket.
+ session := newSession(Monotonic, cluster, cluster.dialInfo)
+ session.setSocket(socket)
+
+ var cmd = bson.D{{Name: "isMaster", Value: 1}}
+
+ // Send client metadata to the server to identify this socket if this is
+ // the first isMaster call only.
+ //
+ // isMaster commands issued after the initial connection handshake MUST NOT contain handshake arguments
+ // https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#connection-handshake
+ //
+ socket.sendMeta.Do(func() {
+ var meta = bson.M{
+ "driver": bson.M{
+ "name": "mgo",
+ "version": "globalsign",
+ },
+ "os": bson.M{
+ "type": runtime.GOOS,
+ "architecture": runtime.GOARCH,
+ },
+ }
+
+ // Include the application name if set
+ if cluster.dialInfo.AppName != "" {
+ meta["application"] = bson.M{"name": cluster.dialInfo.AppName}
+ }
+
+ cmd = append(cmd, bson.DocElem{
+ Name: "client",
+ Value: meta,
+ })
+ })
+
+ err := session.runOnSocket(socket, cmd, result)
+ session.Close()
+ return err
+}
+
+type possibleTimeout interface {
+ Timeout() bool
+}
+
+func (cluster *mongoCluster) syncServer(server *mongoServer) (info *mongoServerInfo, hosts []string, err error) {
+ addr := server.Addr
+ log("SYNC Processing ", addr, "...")
+
+ // Retry a few times to avoid knocking a server down for a hiccup.
+ var result isMasterResult
+ var tryerr error
+ for retry := 0; ; retry++ {
+ if retry == 3 || retry == 1 && cluster.dialInfo.FailFast {
+ return nil, nil, tryerr
+ }
+ if retry > 0 {
+ // Don't abuse the server needlessly if there's something actually wrong.
+ if err, ok := tryerr.(possibleTimeout); ok && err.Timeout() {
+ // Give a chance for waiters to timeout as well.
+ cluster.serverSynced.Broadcast()
+ }
+ time.Sleep(syncShortDelay)
+ }
+
+ // Don't ever hit the pool limit for syncing
+ config := cluster.dialInfo.Copy()
+ config.PoolLimit = 0
+
+ socket, _, err := server.AcquireSocket(config)
+ if err != nil {
+ tryerr = err
+ logf("SYNC Failed to get socket to %s: %v", addr, err)
+ continue
+ }
+ err = cluster.isMaster(socket, &result)
+
+ // Restore the correct dial config before returning it to the pool
+ socket.dialInfo = cluster.dialInfo
+ socket.Release()
+
+ if err != nil {
+ tryerr = err
+ logf("SYNC Command 'ismaster' to %s failed: %v", addr, err)
+ continue
+ }
+ debugf("SYNC Result of 'ismaster' from %s: %#v", addr, result)
+ break
+ }
+
+ if cluster.dialInfo.ReplicaSetName != "" && result.SetName != cluster.dialInfo.ReplicaSetName {
+ logf("SYNC Server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName)
+ return nil, nil, fmt.Errorf("server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName)
+ }
+
+ if result.IsMaster {
+ debugf("SYNC %s is a master.", addr)
+ if !server.info.Master {
+ // Made an incorrect assumption above, so fix stats.
+ stats.conn(-1, false)
+ stats.conn(+1, true)
+ }
+ } else if result.Secondary {
+ debugf("SYNC %s is a slave.", addr)
+ } else if cluster.dialInfo.Direct {
+ logf("SYNC %s in unknown state. Pretending it's a slave due to direct connection.", addr)
+ } else {
+ logf("SYNC %s is neither a master nor a slave.", addr)
+ // Let stats track it as whatever was known before.
+ return nil, nil, errors.New(addr + " is not a master nor slave")
+ }
+
+ info = &mongoServerInfo{
+ Master: result.IsMaster,
+ Mongos: result.Msg == "isdbgrid",
+ Tags: result.Tags,
+ SetName: result.SetName,
+ MaxWireVersion: result.MaxWireVersion,
+ }
+
+ hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives))
+ if result.Primary != "" {
+ // First in the list to speed up master discovery.
+ hosts = append(hosts, result.Primary)
+ }
+ hosts = append(hosts, result.Hosts...)
+ hosts = append(hosts, result.Passives...)
+
+ debugf("SYNC %s knows about the following peers: %#v", addr, hosts)
+ return info, hosts, nil
+}
+
+type syncKind bool
+
+const (
+ completeSync syncKind = true
+ partialSync syncKind = false
+)
+
+func (cluster *mongoCluster) addServer(server *mongoServer, info *mongoServerInfo, syncKind syncKind) {
+ cluster.Lock()
+ current := cluster.servers.Search(server.ResolvedAddr)
+ if current == nil {
+ if syncKind == partialSync {
+ cluster.Unlock()
+ server.Close()
+ log("SYNC Discarding unknown server ", server.Addr, " due to partial sync.")
+ return
+ }
+ cluster.servers.Add(server)
+ if info.Master {
+ cluster.masters.Add(server)
+ log("SYNC Adding ", server.Addr, " to cluster as a master.")
+ } else {
+ log("SYNC Adding ", server.Addr, " to cluster as a slave.")
+ }
+ } else {
+ if server != current {
+ panic("addServer attempting to add duplicated server")
+ }
+ if server.Info().Master != info.Master {
+ if info.Master {
+ log("SYNC Server ", server.Addr, " is now a master.")
+ cluster.masters.Add(server)
+ } else {
+ log("SYNC Server ", server.Addr, " is now a slave.")
+ cluster.masters.Remove(server)
+ }
+ }
+ }
+ server.SetInfo(info)
+ debugf("SYNC Broadcasting availability of server %s", server.Addr)
+ cluster.serverSynced.Broadcast()
+ cluster.Unlock()
+}
+
+func (cluster *mongoCluster) getKnownAddrs() []string {
+ cluster.RLock()
+ max := len(cluster.userSeeds) + len(cluster.dynaSeeds) + cluster.servers.Len()
+ seen := make(map[string]bool, max)
+ known := make([]string, 0, max)
+
+ add := func(addr string) {
+ if _, found := seen[addr]; !found {
+ seen[addr] = true
+ known = append(known, addr)
+ }
+ }
+
+ for _, addr := range cluster.userSeeds {
+ add(addr)
+ }
+ for _, addr := range cluster.dynaSeeds {
+ add(addr)
+ }
+ for _, serv := range cluster.servers.Slice() {
+ add(serv.Addr)
+ }
+ cluster.RUnlock()
+
+ return known
+}
+
+// syncServers injects a value into the cluster.sync channel to force
+// an iteration of the syncServersLoop function.
+func (cluster *mongoCluster) syncServers() {
+ select {
+ case cluster.sync <- true:
+ default:
+ }
+}
+
+// How long to wait for a checkup of the cluster topology if nothing
+// else kicks a synchronization before that.
+const syncServersDelay = 30 * time.Second
+const syncShortDelay = 500 * time.Millisecond
+
+// syncServersLoop loops while the cluster is alive to keep its idea of
+// the server topology up-to-date. It must be called just once from
+// newCluster. The loop iterates once syncServersDelay has passed, or
+// if somebody injects a value into the cluster.sync channel to force a
+// synchronization. A loop iteration will contact all servers in
+// parallel, ask them about known peers and their own role within the
+// cluster, and then attempt to do the same with all the peers
+// retrieved.
+func (cluster *mongoCluster) syncServersLoop() {
+ for {
+ debugf("SYNC Cluster %p is starting a sync loop iteration.", cluster)
+
+ cluster.Lock()
+ if cluster.references == 0 {
+ cluster.Unlock()
+ break
+ }
+ cluster.references++ // Keep alive while syncing.
+ direct := cluster.dialInfo.Direct
+ cluster.Unlock()
+
+ cluster.syncServersIteration(direct)
+
+ // We just synchronized, so consume any outstanding requests.
+ select {
+ case <-cluster.sync:
+ default:
+ }
+
+ cluster.Release()
+
+ // Hold off before allowing another sync. No point in
+ // burning CPU looking for down servers.
+ if !cluster.dialInfo.FailFast {
+ time.Sleep(syncShortDelay)
+ }
+
+ cluster.Lock()
+ if cluster.references == 0 {
+ cluster.Unlock()
+ break
+ }
+ cluster.syncCount++
+ // Poke all waiters so they have a chance to timeout or
+ // restart syncing if they wish to.
+ cluster.serverSynced.Broadcast()
+ // Check if we have to restart immediately either way.
+ restart := !direct && cluster.masters.Empty() || cluster.servers.Empty()
+ cluster.Unlock()
+
+ if restart {
+ log("SYNC No masters found. Will synchronize again.")
+ time.Sleep(syncShortDelay)
+ continue
+ }
+
+ debugf("SYNC Cluster %p waiting for next requested or scheduled sync.", cluster)
+
+ // Hold off until somebody explicitly requests a synchronization
+ // or it's time to check for a cluster topology change again.
+ select {
+ case <-cluster.sync:
+ case <-time.After(syncServersDelay):
+ }
+ }
+ debugf("SYNC Cluster %p is stopping its sync loop.", cluster)
+}
+
+func (cluster *mongoCluster) server(addr string, tcpaddr *net.TCPAddr) *mongoServer {
+ cluster.RLock()
+ server := cluster.servers.Search(tcpaddr.String())
+ cluster.RUnlock()
+ if server != nil {
+ return server
+ }
+ return newServer(addr, tcpaddr, cluster.sync, cluster.dial, cluster.dialInfo)
+}
+
+func resolveAddr(addr string) (*net.TCPAddr, error) {
+ // Simple cases that do not need actual resolution. Works with IPv4 and v6.
+ if host, port, err := net.SplitHostPort(addr); err == nil {
+ if port, _ := strconv.Atoi(port); port > 0 {
+ zone := ""
+ if i := strings.LastIndex(host, "%"); i >= 0 {
+ zone = host[i+1:]
+ host = host[:i]
+ }
+ ip := net.ParseIP(host)
+ if ip != nil {
+ return &net.TCPAddr{IP: ip, Port: port, Zone: zone}, nil
+ }
+ }
+ }
+
+ // Attempt to resolve IPv4 and v6 concurrently.
+ addrChan := make(chan *net.TCPAddr, 2)
+ for _, network := range []string{"udp4", "udp6"} {
+ network := network
+ go func() {
+ // The unfortunate UDP dialing hack allows having a timeout on address resolution.
+ conn, err := net.DialTimeout(network, addr, 10*time.Second)
+ if err != nil {
+ addrChan <- nil
+ } else {
+ addrChan <- (*net.TCPAddr)(conn.RemoteAddr().(*net.UDPAddr))
+ conn.Close()
+ }
+ }()
+ }
+
+ // Wait for the result of IPv4 and v6 resolution. Use IPv4 if available.
+ tcpaddr := <-addrChan
+ if tcpaddr == nil || len(tcpaddr.IP) != 4 {
+ var timeout <-chan time.Time
+ if tcpaddr != nil {
+ // Don't wait too long if an IPv6 address is known.
+ timeout = time.After(50 * time.Millisecond)
+ }
+ select {
+ case <-timeout:
+ case tcpaddr2 := <-addrChan:
+ if tcpaddr == nil || tcpaddr2 != nil {
+ // It's an IPv4 address or the only known address. Use it.
+ tcpaddr = tcpaddr2
+ }
+ }
+ }
+
+ if tcpaddr == nil {
+ log("SYNC Failed to resolve server address: ", addr)
+ return nil, errors.New("failed to resolve server address: " + addr)
+ }
+ if tcpaddr.String() != addr {
+ debug("SYNC Address ", addr, " resolved as ", tcpaddr.String())
+ }
+ return tcpaddr, nil
+}
+
+type pendingAdd struct {
+ server *mongoServer
+ info *mongoServerInfo
+}
+
+func (cluster *mongoCluster) syncServersIteration(direct bool) {
+ log("SYNC Starting full topology synchronization...")
+
+ var wg sync.WaitGroup
+ var m sync.Mutex
+ notYetAdded := make(map[string]pendingAdd)
+ addIfFound := make(map[string]bool)
+ seen := make(map[string]bool)
+ syncKind := partialSync
+
+ var spawnSync func(addr string, byMaster bool)
+ spawnSync = func(addr string, byMaster bool) {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+
+ tcpaddr, err := resolveAddr(addr)
+ if err != nil {
+ log("SYNC Failed to start sync of ", addr, ": ", err.Error())
+ return
+ }
+ resolvedAddr := tcpaddr.String()
+
+ m.Lock()
+ if byMaster {
+ if pending, ok := notYetAdded[resolvedAddr]; ok {
+ delete(notYetAdded, resolvedAddr)
+ m.Unlock()
+ cluster.addServer(pending.server, pending.info, completeSync)
+ return
+ }
+ addIfFound[resolvedAddr] = true
+ }
+ if seen[resolvedAddr] {
+ m.Unlock()
+ return
+ }
+ seen[resolvedAddr] = true
+ m.Unlock()
+
+ server := cluster.server(addr, tcpaddr)
+ info, hosts, err := cluster.syncServer(server)
+ if err != nil {
+ cluster.removeServer(server)
+ return
+ }
+
+ m.Lock()
+ add := direct || info.Master || addIfFound[resolvedAddr]
+ if add {
+ syncKind = completeSync
+ } else {
+ notYetAdded[resolvedAddr] = pendingAdd{server, info}
+ }
+ m.Unlock()
+ if add {
+ cluster.addServer(server, info, completeSync)
+ }
+ if !direct {
+ for _, addr := range hosts {
+ spawnSync(addr, info.Master)
+ }
+ }
+ }()
+ }
+
+ knownAddrs := cluster.getKnownAddrs()
+ for _, addr := range knownAddrs {
+ spawnSync(addr, false)
+ }
+ wg.Wait()
+
+ if syncKind == completeSync {
+ logf("SYNC Synchronization was complete (got data from primary).")
+ for _, pending := range notYetAdded {
+ cluster.removeServer(pending.server)
+ }
+ } else {
+ logf("SYNC Synchronization was partial (cannot talk to primary).")
+ for _, pending := range notYetAdded {
+ cluster.addServer(pending.server, pending.info, partialSync)
+ }
+ }
+
+ cluster.Lock()
+ mastersLen := cluster.masters.Len()
+ logf("SYNC Synchronization completed: %d master(s) and %d slave(s) alive.", mastersLen, cluster.servers.Len()-mastersLen)
+
+ // Update dynamic seeds, but only if we have any good servers. Otherwise,
+ // leave them alone for better chances of a successful sync in the future.
+ if syncKind == completeSync {
+ dynaSeeds := make([]string, cluster.servers.Len())
+ for i, server := range cluster.servers.Slice() {
+ dynaSeeds[i] = server.Addr
+ }
+ cluster.dynaSeeds = dynaSeeds
+ debugf("SYNC New dynamic seeds: %#v\n", dynaSeeds)
+ }
+ cluster.Unlock()
+}
+
+// AcquireSocketWithPoolTimeout returns a socket to a server in the cluster. If slaveOk is
+// true, it will attempt to return a socket to a slave server. If it is
+// false, the socket will necessarily be to a master server.
+func (cluster *mongoCluster) AcquireSocketWithPoolTimeout(mode Mode, slaveOk bool, syncTimeout time.Duration, serverTags []bson.D, info *DialInfo) (s *mongoSocket, err error) {
+ var started time.Time
+ var syncCount uint
+ for {
+ cluster.RLock()
+ for {
+ mastersLen := cluster.masters.Len()
+ slavesLen := cluster.servers.Len() - mastersLen
+ debugf("Cluster has %d known masters and %d known slaves.", mastersLen, slavesLen)
+ if mastersLen > 0 && !(slaveOk && mode == Secondary) || slavesLen > 0 && slaveOk {
+ break
+ }
+ if mastersLen > 0 && mode == Secondary && cluster.masters.HasMongos() {
+ break
+ }
+ if started.IsZero() {
+ // Initialize after fast path above.
+ started = time.Now()
+ syncCount = cluster.syncCount
+ } else if syncTimeout != 0 && started.Before(time.Now().Add(-syncTimeout)) || cluster.dialInfo.FailFast && cluster.syncCount != syncCount {
+ cluster.RUnlock()
+ return nil, errors.New("no reachable servers")
+ }
+ log("Waiting for servers to synchronize...")
+ cluster.syncServers()
+
+ // Remember: this will release and reacquire the lock.
+ cluster.serverSynced.Wait()
+ }
+
+ var server *mongoServer
+ if slaveOk {
+ server = cluster.servers.BestFit(mode, serverTags)
+ } else {
+ server = cluster.masters.BestFit(mode, nil)
+ }
+ cluster.RUnlock()
+
+ if server == nil {
+ // Must have failed the requested tags. Sleep to avoid spinning.
+ time.Sleep(1e8)
+ continue
+ }
+
+ s, abended, err := server.AcquireSocketWithBlocking(info)
+ if err == errPoolTimeout {
+ // No need to remove servers from the topology if acquiring a socket fails for this reason.
+ return nil, err
+ }
+ if err != nil {
+ cluster.removeServer(server)
+ cluster.syncServers()
+ continue
+ }
+ if abended && !slaveOk {
+ var result isMasterResult
+ err := cluster.isMaster(s, &result)
+ if err != nil || !result.IsMaster {
+ logf("Cannot confirm server %s as master (%v)", server.Addr, err)
+ s.Release()
+ cluster.syncServers()
+ time.Sleep(100 * time.Millisecond)
+ continue
+ } else {
+ // We've managed to successfully reconnect to the master, we are no longer abnormaly ended
+ server.Lock()
+ server.abended = false
+ server.Unlock()
+ }
+ }
+ return s, nil
+ }
+}
+
+func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) {
+ cluster.Lock()
+ if cluster.cachedIndex == nil {
+ cluster.cachedIndex = make(map[string]bool)
+ }
+ if exists {
+ cluster.cachedIndex[cacheKey] = true
+ } else {
+ delete(cluster.cachedIndex, cacheKey)
+ }
+ cluster.Unlock()
+}
+
+func (cluster *mongoCluster) HasCachedIndex(cacheKey string) (result bool) {
+ cluster.RLock()
+ if cluster.cachedIndex != nil {
+ result = cluster.cachedIndex[cacheKey]
+ }
+ cluster.RUnlock()
+ return
+}
+
+func (cluster *mongoCluster) ResetIndexCache() {
+ cluster.Lock()
+ cluster.cachedIndex = make(map[string]bool)
+ cluster.Unlock()
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/coarse_time.go b/backend/vendor/github.com/globalsign/mgo/coarse_time.go
new file mode 100644
index 00000000..e54dd17c
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/coarse_time.go
@@ -0,0 +1,62 @@
+package mgo
+
+import (
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// coarseTimeProvider provides a periodically updated (approximate) time value to
+// amortise the cost of frequent calls to time.Now.
+//
+// A read throughput increase of ~6% was measured when using coarseTimeProvider with the
+// high-precision event timer (HPET) on FreeBSD 11.1 and Go 1.10.1 after merging
+// #116.
+//
+// Calling Now returns a time.Time that is updated at the configured interval,
+// however due to scheduling the value may be marginally older than expected.
+//
+// coarseTimeProvider is safe for concurrent use.
+type coarseTimeProvider struct {
+ once sync.Once
+ stop chan struct{}
+ last atomic.Value
+}
+
+// Now returns the most recently acquired time.Time value.
+func (t *coarseTimeProvider) Now() time.Time {
+ return t.last.Load().(time.Time)
+}
+
+// Close stops the periodic update of t.
+//
+// Any subsequent calls to Now will return the same value forever.
+func (t *coarseTimeProvider) Close() {
+ t.once.Do(func() {
+ close(t.stop)
+ })
+}
+
+// newcoarseTimeProvider returns a coarseTimeProvider configured to update at granularity.
+func newcoarseTimeProvider(granularity time.Duration) *coarseTimeProvider {
+ t := &coarseTimeProvider{
+ stop: make(chan struct{}),
+ }
+
+ t.last.Store(time.Now())
+
+ go func() {
+ ticker := time.NewTicker(granularity)
+ for {
+ select {
+ case <-t.stop:
+ ticker.Stop()
+ return
+ case <-ticker.C:
+ t.last.Store(time.Now())
+ }
+ }
+ }()
+
+ return t
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/doc.go b/backend/vendor/github.com/globalsign/mgo/doc.go
new file mode 100644
index 00000000..f3f373bf
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/doc.go
@@ -0,0 +1,35 @@
+// Package mgo (pronounced as "mango") offers a rich MongoDB driver for Go.
+//
+// Detailed documentation of the API is available at GoDoc:
+//
+// https://godoc.org/github.com/globalsign/mgo
+//
+// Usage of the driver revolves around the concept of sessions. To
+// get started, obtain a session using the Dial function:
+//
+// session, err := mgo.Dial(url)
+//
+// This will establish one or more connections with the cluster of
+// servers defined by the url parameter. From then on, the cluster
+// may be queried with multiple consistency rules (see SetMode) and
+// documents retrieved with statements such as:
+//
+// c := session.DB(database).C(collection)
+// err := c.Find(query).One(&result)
+//
+// New sessions are typically created by calling session.Copy on the
+// initial session obtained at dial time. These new sessions will share
+// the same cluster information and connection pool, and may be easily
+// handed into other methods and functions for organizing logic.
+// Every session created must have its Close method called at the end
+// of its life time, so its resources may be put back in the pool or
+// collected, depending on the case.
+//
+// There is a sub-package that provides support for BSON, which can be
+// used by itself as well:
+//
+// https://godoc.org/github.com/globalsign/mgo/bson
+//
+// For more details, see the documentation for the types and methods.
+//
+package mgo
diff --git a/backend/vendor/github.com/globalsign/mgo/gridfs.go b/backend/vendor/github.com/globalsign/mgo/gridfs.go
new file mode 100644
index 00000000..0954b166
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/gridfs.go
@@ -0,0 +1,782 @@
+// mgo - MongoDB driver for Go
+//
+// Copyright (c) 2010-2012 - Gustavo Niemeyer
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package mgo
+
+import (
+ "crypto/md5"
+ "encoding/hex"
+ "errors"
+ "hash"
+ "io"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/globalsign/mgo/bson"
+)
+
+// GridFS stores files in two collections:
+//
+// - chunks stores the binary chunks. For details, see the chunks Collection.
+// - files stores the file’s metadata. For details, see the files Collection.
+//
+// GridFS places the collections in a common bucket by prefixing each with the bucket name.
+// By default, GridFS uses two collections with a bucket named fs:
+//
+// - fs.files
+// - fs.chunks
+//
+// You can choose a different bucket name, as well as create multiple buckets in a single database.
+// The full collection name, which includes the bucket name, is subject to the namespace length limit.
+//
+// Relevant documentation:
+//
+// https://docs.mongodb.com/manual/core/gridfs/
+// https://docs.mongodb.com/manual/core/gridfs/#gridfs-chunks-collection
+// https://docs.mongodb.com/manual/core/gridfs/#gridfs-files-collection
+//
+type GridFS struct {
+ Files *Collection
+ Chunks *Collection
+}
+
+type gfsFileMode int
+
+const (
+ gfsClosed gfsFileMode = 0
+ gfsReading gfsFileMode = 1
+ gfsWriting gfsFileMode = 2
+)
+
+// GridFile document in files collection
+type GridFile struct {
+ m sync.Mutex
+ c sync.Cond
+ gfs *GridFS
+ mode gfsFileMode
+ err error
+
+ chunk int
+ offset int64
+
+ wpending int
+ wbuf []byte
+ wsum hash.Hash
+
+ rbuf []byte
+ rcache *gfsCachedChunk
+
+ doc gfsFile
+}
+
+type gfsFile struct {
+ Id interface{} `bson:"_id"`
+ ChunkSize int `bson:"chunkSize"`
+ UploadDate time.Time `bson:"uploadDate"`
+ Length int64 `bson:",minsize"`
+ MD5 string
+ Filename string `bson:",omitempty"`
+ ContentType string `bson:"contentType,omitempty"`
+ Metadata *bson.Raw `bson:",omitempty"`
+}
+
+type gfsChunk struct {
+ Id interface{} `bson:"_id"`
+ FilesId interface{} `bson:"files_id"`
+ N int
+ Data []byte
+}
+
+type gfsCachedChunk struct {
+ wait sync.Mutex
+ n int
+ data []byte
+ err error
+}
+
+func newGridFS(db *Database, prefix string) *GridFS {
+ return &GridFS{db.C(prefix + ".files"), db.C(prefix + ".chunks")}
+}
+
+func (gfs *GridFS) newFile() *GridFile {
+ file := &GridFile{gfs: gfs}
+ file.c.L = &file.m
+ //runtime.SetFinalizer(file, finalizeFile)
+ return file
+}
+
+func finalizeFile(file *GridFile) {
+ file.Close()
+}
+
+// Create creates a new file with the provided name in the GridFS. If the file
+// name already exists, a new version will be inserted with an up-to-date
+// uploadDate that will cause it to be atomically visible to the Open and
+// OpenId methods. If the file name is not important, an empty name may be
+// provided and the file Id used instead.
+//
+// It's important to Close files whether they are being written to
+// or read from, and to check the err result to ensure the operation
+// completed successfully.
+//
+// A simple example inserting a new file:
+//
+// func check(err error) {
+// if err != nil {
+// panic(err.String())
+// }
+// }
+// file, err := db.GridFS("fs").Create("myfile.txt")
+// check(err)
+// n, err := file.Write([]byte("Hello world!"))
+// check(err)
+// err = file.Close()
+// check(err)
+// fmt.Printf("%d bytes written\n", n)
+//
+// The io.Writer interface is implemented by *GridFile and may be used to
+// help on the file creation. For example:
+//
+// file, err := db.GridFS("fs").Create("myfile.txt")
+// check(err)
+// messages, err := os.Open("/var/log/messages")
+// check(err)
+// defer messages.Close()
+// err = io.Copy(file, messages)
+// check(err)
+// err = file.Close()
+// check(err)
+//
+func (gfs *GridFS) Create(name string) (file *GridFile, err error) {
+ file = gfs.newFile()
+ file.mode = gfsWriting
+ file.wsum = md5.New()
+ file.doc = gfsFile{Id: bson.NewObjectId(), ChunkSize: 255 * 1024, Filename: name}
+ return
+}
+
+// OpenId returns the file with the provided id, for reading.
+// If the file isn't found, err will be set to mgo.ErrNotFound.
+//
+// It's important to Close files whether they are being written to
+// or read from, and to check the err result to ensure the operation
+// completed successfully.
+//
+// The following example will print the first 8192 bytes from the file:
+//
+// func check(err error) {
+// if err != nil {
+// panic(err.String())
+// }
+// }
+// file, err := db.GridFS("fs").OpenId(objid)
+// check(err)
+// b := make([]byte, 8192)
+// n, err := file.Read(b)
+// check(err)
+// fmt.Println(string(b))
+// check(err)
+// err = file.Close()
+// check(err)
+// fmt.Printf("%d bytes read\n", n)
+//
+// The io.Reader interface is implemented by *GridFile and may be used to
+// deal with it. As an example, the following snippet will dump the whole
+// file into the standard output:
+//
+// file, err := db.GridFS("fs").OpenId(objid)
+// check(err)
+// err = io.Copy(os.Stdout, file)
+// check(err)
+// err = file.Close()
+// check(err)
+//
+func (gfs *GridFS) OpenId(id interface{}) (file *GridFile, err error) {
+ var doc gfsFile
+ err = gfs.Files.Find(bson.M{"_id": id}).One(&doc)
+ if err != nil {
+ return
+ }
+ file = gfs.newFile()
+ file.mode = gfsReading
+ file.doc = doc
+ return
+}
+
+// Open returns the most recently uploaded file with the provided
+// name, for reading. If the file isn't found, err will be set
+// to mgo.ErrNotFound.
+//
+// It's important to Close files whether they are being written to
+// or read from, and to check the err result to ensure the operation
+// completed successfully.
+//
+// The following example will print the first 8192 bytes from the file:
+//
+// file, err := db.GridFS("fs").Open("myfile.txt")
+// check(err)
+// b := make([]byte, 8192)
+// n, err := file.Read(b)
+// check(err)
+// fmt.Println(string(b))
+// check(err)
+// err = file.Close()
+// check(err)
+// fmt.Printf("%d bytes read\n", n)
+//
+// The io.Reader interface is implemented by *GridFile and may be used to
+// deal with it. As an example, the following snippet will dump the whole
+// file into the standard output:
+//
+// file, err := db.GridFS("fs").Open("myfile.txt")
+// check(err)
+// err = io.Copy(os.Stdout, file)
+// check(err)
+// err = file.Close()
+// check(err)
+//
+func (gfs *GridFS) Open(name string) (file *GridFile, err error) {
+ var doc gfsFile
+ err = gfs.Files.Find(bson.M{"filename": name}).Sort("-uploadDate").One(&doc)
+ if err != nil {
+ return
+ }
+ file = gfs.newFile()
+ file.mode = gfsReading
+ file.doc = doc
+ return
+}
+
+// OpenNext opens the next file from iter for reading, sets *file to it,
+// and returns true on the success case. If no more documents are available
+// on iter or an error occurred, *file is set to nil and the result is false.
+// Errors will be available via iter.Err().
+//
+// The iter parameter must be an iterator on the GridFS files collection.
+// Using the GridFS.Find method is an easy way to obtain such an iterator,
+// but any iterator on the collection will work.
+//
+// If the provided *file is non-nil, OpenNext will close it before attempting
+// to iterate to the next element. This means that in a loop one only
+// has to worry about closing files when breaking out of the loop early
+// (break, return, or panic).
+//
+// For example:
+//
+// gfs := db.GridFS("fs")
+// query := gfs.Find(nil).Sort("filename")
+// iter := query.Iter()
+// var f *mgo.GridFile
+// for gfs.OpenNext(iter, &f) {
+// fmt.Printf("Filename: %s\n", f.Name())
+// }
+// if iter.Close() != nil {
+// panic(iter.Close())
+// }
+//
+func (gfs *GridFS) OpenNext(iter *Iter, file **GridFile) bool {
+ if *file != nil {
+ // Ignoring the error here shouldn't be a big deal
+ // as we're reading the file and the loop iteration
+ // for this file is finished.
+ _ = (*file).Close()
+ }
+ var doc gfsFile
+ if !iter.Next(&doc) {
+ *file = nil
+ return false
+ }
+ f := gfs.newFile()
+ f.mode = gfsReading
+ f.doc = doc
+ *file = f
+ return true
+}
+
+// Find runs query on GridFS's files collection and returns
+// the resulting Query.
+//
+// This logic:
+//
+// gfs := db.GridFS("fs")
+// iter := gfs.Find(nil).Iter()
+//
+// Is equivalent to:
+//
+// files := db.C("fs" + ".files")
+// iter := files.Find(nil).Iter()
+//
+func (gfs *GridFS) Find(query interface{}) *Query {
+ return gfs.Files.Find(query)
+}
+
+// RemoveId deletes the file with the provided id from the GridFS.
+func (gfs *GridFS) RemoveId(id interface{}) error {
+ err := gfs.Files.Remove(bson.M{"_id": id})
+ if err != nil {
+ return err
+ }
+ _, err = gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: id}})
+ return err
+}
+
+type gfsDocId struct {
+ Id interface{} `bson:"_id"`
+}
+
+// Remove deletes all files with the provided name from the GridFS.
+func (gfs *GridFS) Remove(name string) (err error) {
+ iter := gfs.Files.Find(bson.M{"filename": name}).Select(bson.M{"_id": 1}).Iter()
+ var doc gfsDocId
+ for iter.Next(&doc) {
+ if e := gfs.RemoveId(doc.Id); e != nil {
+ err = e
+ }
+ }
+ if err == nil {
+ err = iter.Close()
+ }
+ return err
+}
+
+func (file *GridFile) assertMode(mode gfsFileMode) {
+ switch file.mode {
+ case mode:
+ return
+ case gfsWriting:
+ panic("GridFile is open for writing")
+ case gfsReading:
+ panic("GridFile is open for reading")
+ case gfsClosed:
+ panic("GridFile is closed")
+ default:
+ panic("internal error: missing GridFile mode")
+ }
+}
+
+// SetChunkSize sets size of saved chunks. Once the file is written to, it
+// will be split in blocks of that size and each block saved into an
+// independent chunk document. The default chunk size is 255kb.
+//
+// It is a runtime error to call this function once the file has started
+// being written to.
+func (file *GridFile) SetChunkSize(bytes int) {
+ file.assertMode(gfsWriting)
+ debugf("GridFile %p: setting chunk size to %d", file, bytes)
+ file.m.Lock()
+ file.doc.ChunkSize = bytes
+ file.m.Unlock()
+}
+
+// Id returns the current file Id.
+func (file *GridFile) Id() interface{} {
+ return file.doc.Id
+}
+
+// SetId changes the current file Id.
+//
+// It is a runtime error to call this function once the file has started
+// being written to, or when the file is not open for writing.
+func (file *GridFile) SetId(id interface{}) {
+ file.assertMode(gfsWriting)
+ file.m.Lock()
+ file.doc.Id = id
+ file.m.Unlock()
+}
+
+// Name returns the optional file name. An empty string will be returned
+// in case it is unset.
+func (file *GridFile) Name() string {
+ return file.doc.Filename
+}
+
+// SetName changes the optional file name. An empty string may be used to
+// unset it.
+//
+// It is a runtime error to call this function when the file is not open
+// for writing.
+func (file *GridFile) SetName(name string) {
+ file.assertMode(gfsWriting)
+ file.m.Lock()
+ file.doc.Filename = name
+ file.m.Unlock()
+}
+
+// ContentType returns the optional file content type. An empty string will be
+// returned in case it is unset.
+func (file *GridFile) ContentType() string {
+ return file.doc.ContentType
+}
+
+// SetContentType changes the optional file content type. An empty string may be
+// used to unset it.
+//
+// It is a runtime error to call this function when the file is not open
+// for writing.
+func (file *GridFile) SetContentType(ctype string) {
+ file.assertMode(gfsWriting)
+ file.m.Lock()
+ file.doc.ContentType = ctype
+ file.m.Unlock()
+}
+
+// GetMeta unmarshals the optional "metadata" field associated with the
+// file into the result parameter. The meaning of keys under that field
+// is user-defined. For example:
+//
+// result := struct{ INode int }{}
+// err = file.GetMeta(&result)
+// if err != nil {
+// panic(err.String())
+// }
+// fmt.Printf("inode: %d\n", result.INode)
+//
+func (file *GridFile) GetMeta(result interface{}) (err error) {
+ file.m.Lock()
+ if file.doc.Metadata != nil {
+ err = bson.Unmarshal(file.doc.Metadata.Data, result)
+ }
+ file.m.Unlock()
+ return
+}
+
+// SetMeta changes the optional "metadata" field associated with the
+// file. The meaning of keys under that field is user-defined.
+// For example:
+//
+// file.SetMeta(bson.M{"inode": inode})
+//
+// It is a runtime error to call this function when the file is not open
+// for writing.
+func (file *GridFile) SetMeta(metadata interface{}) {
+ file.assertMode(gfsWriting)
+ data, err := bson.Marshal(metadata)
+ file.m.Lock()
+ if err != nil && file.err == nil {
+ file.err = err
+ } else {
+ file.doc.Metadata = &bson.Raw{Data: data}
+ }
+ file.m.Unlock()
+}
+
+// Size returns the file size in bytes.
+func (file *GridFile) Size() (bytes int64) {
+ file.m.Lock()
+ bytes = file.doc.Length
+ file.m.Unlock()
+ return
+}
+
+// MD5 returns the file MD5 as a hex-encoded string.
+func (file *GridFile) MD5() (md5 string) {
+ return file.doc.MD5
+}
+
+// UploadDate returns the file upload time.
+func (file *GridFile) UploadDate() time.Time {
+ return file.doc.UploadDate
+}
+
+// SetUploadDate changes the file upload time.
+//
+// It is a runtime error to call this function when the file is not open
+// for writing.
+func (file *GridFile) SetUploadDate(t time.Time) {
+ file.assertMode(gfsWriting)
+ file.m.Lock()
+ file.doc.UploadDate = t
+ file.m.Unlock()
+}
+
+// Close flushes any pending changes in case the file is being written
+// to, waits for any background operations to finish, and closes the file.
+//
+// It's important to Close files whether they are being written to
+// or read from, and to check the err result to ensure the operation
+// completed successfully.
+func (file *GridFile) Close() (err error) {
+ file.m.Lock()
+ defer file.m.Unlock()
+ if file.mode == gfsWriting {
+ if len(file.wbuf) > 0 && file.err == nil {
+ file.insertChunk(file.wbuf)
+ file.wbuf = file.wbuf[0:0]
+ }
+ file.completeWrite()
+ } else if file.mode == gfsReading && file.rcache != nil {
+ file.rcache.wait.Lock()
+ file.rcache = nil
+ }
+ file.mode = gfsClosed
+ debugf("GridFile %p: closed", file)
+ return file.err
+}
+
+func (file *GridFile) completeWrite() {
+ for file.wpending > 0 {
+ debugf("GridFile %p: waiting for %d pending chunks to complete file write", file, file.wpending)
+ file.c.Wait()
+ }
+ if file.err == nil {
+ hexsum := hex.EncodeToString(file.wsum.Sum(nil))
+ if file.doc.UploadDate.IsZero() {
+ file.doc.UploadDate = bson.Now()
+ }
+ file.doc.MD5 = hexsum
+ file.err = file.gfs.Files.Insert(file.doc)
+ }
+ if file.err != nil {
+ file.gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: file.doc.Id}})
+ }
+ if file.err == nil {
+ index := Index{
+ Key: []string{"files_id", "n"},
+ Unique: true,
+ }
+ file.err = file.gfs.Chunks.EnsureIndex(index)
+ }
+}
+
+// Abort cancels an in-progress write, preventing the file from being
+// automically created and ensuring previously written chunks are
+// removed when the file is closed.
+//
+// It is a runtime error to call Abort when the file was not opened
+// for writing.
+func (file *GridFile) Abort() {
+ if file.mode != gfsWriting {
+ panic("file.Abort must be called on file opened for writing")
+ }
+ file.err = errors.New("write aborted")
+}
+
+// Write writes the provided data to the file and returns the
+// number of bytes written and an error in case something
+// wrong happened.
+//
+// The file will internally cache the data so that all but the last
+// chunk sent to the database have the size defined by SetChunkSize.
+// This also means that errors may be deferred until a future call
+// to Write or Close.
+//
+// The parameters and behavior of this function turn the file
+// into an io.Writer.
+func (file *GridFile) Write(data []byte) (n int, err error) {
+ file.assertMode(gfsWriting)
+ file.m.Lock()
+ debugf("GridFile %p: writing %d bytes", file, len(data))
+ defer file.m.Unlock()
+
+ if file.err != nil {
+ return 0, file.err
+ }
+
+ n = len(data)
+ file.doc.Length += int64(n)
+ chunkSize := file.doc.ChunkSize
+
+ if len(file.wbuf)+len(data) < chunkSize {
+ file.wbuf = append(file.wbuf, data...)
+ return
+ }
+
+ // First, flush file.wbuf complementing with data.
+ if len(file.wbuf) > 0 {
+ missing := chunkSize - len(file.wbuf)
+ if missing > len(data) {
+ missing = len(data)
+ }
+ file.wbuf = append(file.wbuf, data[:missing]...)
+ data = data[missing:]
+ file.insertChunk(file.wbuf)
+ file.wbuf = file.wbuf[0:0]
+ }
+
+ // Then, flush all chunks from data without copying.
+ for len(data) > chunkSize {
+ size := chunkSize
+ if size > len(data) {
+ size = len(data)
+ }
+ file.insertChunk(data[:size])
+ data = data[size:]
+ }
+
+ // And append the rest for a future call.
+ file.wbuf = append(file.wbuf, data...)
+
+ return n, file.err
+}
+
+func (file *GridFile) insertChunk(data []byte) {
+ n := file.chunk
+ file.chunk++
+ debugf("GridFile %p: adding to checksum: %q", file, string(data))
+ file.wsum.Write(data)
+
+ for file.doc.ChunkSize*file.wpending >= 1024*1024 {
+ // Hold on.. we got a MB pending.
+ file.c.Wait()
+ if file.err != nil {
+ return
+ }
+ }
+
+ file.wpending++
+
+ debugf("GridFile %p: inserting chunk %d with %d bytes", file, n, len(data))
+
+ // We may not own the memory of data, so rather than
+ // simply copying it, we'll marshal the document ahead of time.
+ data, err := bson.Marshal(gfsChunk{bson.NewObjectId(), file.doc.Id, n, data})
+ if err != nil {
+ file.err = err
+ return
+ }
+
+ go func() {
+ err := file.gfs.Chunks.Insert(bson.Raw{Data: data})
+ file.m.Lock()
+ file.wpending--
+ if err != nil && file.err == nil {
+ file.err = err
+ }
+ file.c.Broadcast()
+ file.m.Unlock()
+ }()
+}
+
+// Seek sets the offset for the next Read or Write on file to
+// offset, interpreted according to whence: 0 means relative to
+// the origin of the file, 1 means relative to the current offset,
+// and 2 means relative to the end. It returns the new offset and
+// an error, if any.
+func (file *GridFile) Seek(offset int64, whence int) (pos int64, err error) {
+ file.m.Lock()
+ debugf("GridFile %p: seeking for %s (whence=%d)", file, offset, whence)
+ defer file.m.Unlock()
+ switch whence {
+ case os.SEEK_SET:
+ case os.SEEK_CUR:
+ offset += file.offset
+ case os.SEEK_END:
+ offset += file.doc.Length
+ default:
+ panic("unsupported whence value")
+ }
+ if offset > file.doc.Length {
+ return file.offset, errors.New("seek past end of file")
+ }
+ if offset == file.doc.Length {
+ // If we're seeking to the end of the file,
+ // no need to read anything. This enables
+ // a client to find the size of the file using only the
+ // io.ReadSeeker interface with low overhead.
+ file.offset = offset
+ return file.offset, nil
+ }
+ chunk := int(offset / int64(file.doc.ChunkSize))
+ if chunk+1 == file.chunk && offset >= file.offset {
+ file.rbuf = file.rbuf[int(offset-file.offset):]
+ file.offset = offset
+ return file.offset, nil
+ }
+ file.offset = offset
+ file.chunk = chunk
+ file.rbuf = nil
+ file.rbuf, err = file.getChunk()
+ if err == nil {
+ file.rbuf = file.rbuf[int(file.offset-int64(chunk)*int64(file.doc.ChunkSize)):]
+ }
+ return file.offset, err
+}
+
+// Read reads into b the next available data from the file and
+// returns the number of bytes written and an error in case
+// something wrong happened. At the end of the file, n will
+// be zero and err will be set to io.EOF.
+//
+// The parameters and behavior of this function turn the file
+// into an io.Reader.
+func (file *GridFile) Read(b []byte) (n int, err error) {
+ file.assertMode(gfsReading)
+ file.m.Lock()
+ debugf("GridFile %p: reading at offset %d into buffer of length %d", file, file.offset, len(b))
+ defer file.m.Unlock()
+ if file.offset == file.doc.Length {
+ return 0, io.EOF
+ }
+ for err == nil {
+ i := copy(b, file.rbuf)
+ n += i
+ file.offset += int64(i)
+ file.rbuf = file.rbuf[i:]
+ if i == len(b) || file.offset == file.doc.Length {
+ break
+ }
+ b = b[i:]
+ file.rbuf, err = file.getChunk()
+ }
+ return n, err
+}
+
+func (file *GridFile) getChunk() (data []byte, err error) {
+ cache := file.rcache
+ file.rcache = nil
+ if cache != nil && cache.n == file.chunk {
+ debugf("GridFile %p: Getting chunk %d from cache", file, file.chunk)
+ cache.wait.Lock()
+ data, err = cache.data, cache.err
+ } else {
+ debugf("GridFile %p: Fetching chunk %d", file, file.chunk)
+ var doc gfsChunk
+ err = file.gfs.Chunks.Find(bson.D{{Name: "files_id", Value: file.doc.Id}, {Name: "n", Value: file.chunk}}).One(&doc)
+ data = doc.Data
+ }
+ file.chunk++
+ if int64(file.chunk)*int64(file.doc.ChunkSize) < file.doc.Length {
+ // Read the next one in background.
+ cache = &gfsCachedChunk{n: file.chunk}
+ cache.wait.Lock()
+ debugf("GridFile %p: Scheduling chunk %d for background caching", file, file.chunk)
+ // Clone the session to avoid having it closed in between.
+ chunks := file.gfs.Chunks
+ session := chunks.Database.Session.Clone()
+ go func(id interface{}, n int) {
+ defer session.Close()
+ chunks = chunks.With(session)
+ var doc gfsChunk
+ cache.err = chunks.Find(bson.D{{Name: "files_id", Value: id}, {Name: "n", Value: n}}).One(&doc)
+ cache.data = doc.Data
+ cache.wait.Unlock()
+ }(file.doc.Id, file.chunk)
+ file.rcache = cache
+ }
+ debugf("Returning err: %#v", err)
+ return
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/internal/json/LICENSE b/backend/vendor/github.com/globalsign/mgo/internal/json/LICENSE
new file mode 100644
index 00000000..74487567
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/internal/json/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/backend/vendor/github.com/globalsign/mgo/internal/json/decode.go b/backend/vendor/github.com/globalsign/mgo/internal/json/decode.go
new file mode 100644
index 00000000..d5ca1f9a
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/internal/json/decode.go
@@ -0,0 +1,1685 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Represents JSON data structure using native Go types: booleans, floats,
+// strings, arrays, and maps.
+
+package json
+
+import (
+ "bytes"
+ "encoding"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "reflect"
+ "runtime"
+ "strconv"
+ "unicode"
+ "unicode/utf16"
+ "unicode/utf8"
+)
+
+// Unmarshal parses the JSON-encoded data and stores the result
+// in the value pointed to by v.
+//
+// Unmarshal uses the inverse of the encodings that
+// Marshal uses, allocating maps, slices, and pointers as necessary,
+// with the following additional rules:
+//
+// To unmarshal JSON into a pointer, Unmarshal first handles the case of
+// the JSON being the JSON literal null. In that case, Unmarshal sets
+// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into
+// the value pointed at by the pointer. If the pointer is nil, Unmarshal
+// allocates a new value for it to point to.
+//
+// To unmarshal JSON into a struct, Unmarshal matches incoming object
+// keys to the keys used by Marshal (either the struct field name or its tag),
+// preferring an exact match but also accepting a case-insensitive match.
+// Unmarshal will only set exported fields of the struct.
+//
+// To unmarshal JSON into an interface value,
+// Unmarshal stores one of these in the interface value:
+//
+// bool, for JSON booleans
+// float64, for JSON numbers
+// string, for JSON strings
+// []interface{}, for JSON arrays
+// map[string]interface{}, for JSON objects
+// nil for JSON null
+//
+// To unmarshal a JSON array into a slice, Unmarshal resets the slice length
+// to zero and then appends each element to the slice.
+// As a special case, to unmarshal an empty JSON array into a slice,
+// Unmarshal replaces the slice with a new empty slice.
+//
+// To unmarshal a JSON array into a Go array, Unmarshal decodes
+// JSON array elements into corresponding Go array elements.
+// If the Go array is smaller than the JSON array,
+// the additional JSON array elements are discarded.
+// If the JSON array is smaller than the Go array,
+// the additional Go array elements are set to zero values.
+//
+// To unmarshal a JSON object into a map, Unmarshal first establishes a map to
+// use, If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal
+// reuses the existing map, keeping existing entries. Unmarshal then stores key-
+// value pairs from the JSON object into the map. The map's key type must
+// either be a string or implement encoding.TextUnmarshaler.
+//
+// If a JSON value is not appropriate for a given target type,
+// or if a JSON number overflows the target type, Unmarshal
+// skips that field and completes the unmarshaling as best it can.
+// If no more serious errors are encountered, Unmarshal returns
+// an UnmarshalTypeError describing the earliest such error.
+//
+// The JSON null value unmarshals into an interface, map, pointer, or slice
+// by setting that Go value to nil. Because null is often used in JSON to mean
+// ``not present,'' unmarshaling a JSON null into any other Go type has no effect
+// on the value and produces no error.
+//
+// When unmarshaling quoted strings, invalid UTF-8 or
+// invalid UTF-16 surrogate pairs are not treated as an error.
+// Instead, they are replaced by the Unicode replacement
+// character U+FFFD.
+//
+func Unmarshal(data []byte, v interface{}) error {
+ // Check for well-formedness.
+ // Avoids filling out half a data structure
+ // before discovering a JSON syntax error.
+ var d decodeState
+ err := checkValid(data, &d.scan)
+ if err != nil {
+ return err
+ }
+
+ d.init(data)
+ return d.unmarshal(v)
+}
+
+// Unmarshaler is the interface implemented by types
+// that can unmarshal a JSON description of themselves.
+// The input can be assumed to be a valid encoding of
+// a JSON value. UnmarshalJSON must copy the JSON data
+// if it wishes to retain the data after returning.
+type Unmarshaler interface {
+ UnmarshalJSON([]byte) error
+}
+
+// An UnmarshalTypeError describes a JSON value that was
+// not appropriate for a value of a specific Go type.
+type UnmarshalTypeError struct {
+ Value string // description of JSON value - "bool", "array", "number -5"
+ Type reflect.Type // type of Go value it could not be assigned to
+ Offset int64 // error occurred after reading Offset bytes
+}
+
+func (e *UnmarshalTypeError) Error() string {
+ return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String()
+}
+
+// An UnmarshalFieldError describes a JSON object key that
+// led to an unexported (and therefore unwritable) struct field.
+// (No longer used; kept for compatibility.)
+type UnmarshalFieldError struct {
+ Key string
+ Type reflect.Type
+ Field reflect.StructField
+}
+
+func (e *UnmarshalFieldError) Error() string {
+ return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String()
+}
+
+// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal.
+// (The argument to Unmarshal must be a non-nil pointer.)
+type InvalidUnmarshalError struct {
+ Type reflect.Type
+}
+
+func (e *InvalidUnmarshalError) Error() string {
+ if e.Type == nil {
+ return "json: Unmarshal(nil)"
+ }
+
+ if e.Type.Kind() != reflect.Ptr {
+ return "json: Unmarshal(non-pointer " + e.Type.String() + ")"
+ }
+ return "json: Unmarshal(nil " + e.Type.String() + ")"
+}
+
+func (d *decodeState) unmarshal(v interface{}) (err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ if _, ok := r.(runtime.Error); ok {
+ panic(r)
+ }
+ err = r.(error)
+ }
+ }()
+
+ rv := reflect.ValueOf(v)
+ if rv.Kind() != reflect.Ptr || rv.IsNil() {
+ return &InvalidUnmarshalError{reflect.TypeOf(v)}
+ }
+
+ d.scan.reset()
+ // We decode rv not rv.Elem because the Unmarshaler interface
+ // test must be applied at the top level of the value.
+ d.value(rv)
+ return d.savedError
+}
+
+// A Number represents a JSON number literal.
+type Number string
+
+// String returns the literal text of the number.
+func (n Number) String() string { return string(n) }
+
+// Float64 returns the number as a float64.
+func (n Number) Float64() (float64, error) {
+ return strconv.ParseFloat(string(n), 64)
+}
+
+// Int64 returns the number as an int64.
+func (n Number) Int64() (int64, error) {
+ return strconv.ParseInt(string(n), 10, 64)
+}
+
+// isValidNumber reports whether s is a valid JSON number literal.
+func isValidNumber(s string) bool {
+ // This function implements the JSON numbers grammar.
+ // See https://tools.ietf.org/html/rfc7159#section-6
+ // and http://json.org/number.gif
+
+ if s == "" {
+ return false
+ }
+
+ // Optional -
+ if s[0] == '-' {
+ s = s[1:]
+ if s == "" {
+ return false
+ }
+ }
+
+ // Digits
+ switch {
+ default:
+ return false
+
+ case s[0] == '0':
+ s = s[1:]
+
+ case '1' <= s[0] && s[0] <= '9':
+ s = s[1:]
+ for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
+ s = s[1:]
+ }
+ }
+
+ // . followed by 1 or more digits.
+ if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' {
+ s = s[2:]
+ for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
+ s = s[1:]
+ }
+ }
+
+ // e or E followed by an optional - or + and
+ // 1 or more digits.
+ if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') {
+ s = s[1:]
+ if s[0] == '+' || s[0] == '-' {
+ s = s[1:]
+ if s == "" {
+ return false
+ }
+ }
+ for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
+ s = s[1:]
+ }
+ }
+
+ // Make sure we are at the end.
+ return s == ""
+}
+
+// decodeState represents the state while decoding a JSON value.
+type decodeState struct {
+ data []byte
+ off int // read offset in data
+ scan scanner
+ nextscan scanner // for calls to nextValue
+ savedError error
+ useNumber bool
+ ext Extension
+}
+
+// errPhase is used for errors that should not happen unless
+// there is a bug in the JSON decoder or something is editing
+// the data slice while the decoder executes.
+var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?")
+
+func (d *decodeState) init(data []byte) *decodeState {
+ d.data = data
+ d.off = 0
+ d.savedError = nil
+ return d
+}
+
+// error aborts the decoding by panicking with err.
+func (d *decodeState) error(err error) {
+ panic(err)
+}
+
+// saveError saves the first err it is called with,
+// for reporting at the end of the unmarshal.
+func (d *decodeState) saveError(err error) {
+ if d.savedError == nil {
+ d.savedError = err
+ }
+}
+
+// next cuts off and returns the next full JSON value in d.data[d.off:].
+// The next value is known to be an object or array, not a literal.
+func (d *decodeState) next() []byte {
+ c := d.data[d.off]
+ item, rest, err := nextValue(d.data[d.off:], &d.nextscan)
+ if err != nil {
+ d.error(err)
+ }
+ d.off = len(d.data) - len(rest)
+
+ // Our scanner has seen the opening brace/bracket
+ // and thinks we're still in the middle of the object.
+ // invent a closing brace/bracket to get it out.
+ if c == '{' {
+ d.scan.step(&d.scan, '}')
+ } else if c == '[' {
+ d.scan.step(&d.scan, ']')
+ } else {
+ // Was inside a function name. Get out of it.
+ d.scan.step(&d.scan, '(')
+ d.scan.step(&d.scan, ')')
+ }
+
+ return item
+}
+
+// scanWhile processes bytes in d.data[d.off:] until it
+// receives a scan code not equal to op.
+// It updates d.off and returns the new scan code.
+func (d *decodeState) scanWhile(op int) int {
+ var newOp int
+ for {
+ if d.off >= len(d.data) {
+ newOp = d.scan.eof()
+ d.off = len(d.data) + 1 // mark processed EOF with len+1
+ } else {
+ c := d.data[d.off]
+ d.off++
+ newOp = d.scan.step(&d.scan, c)
+ }
+ if newOp != op {
+ break
+ }
+ }
+ return newOp
+}
+
+// value decodes a JSON value from d.data[d.off:] into the value.
+// it updates d.off to point past the decoded value.
+func (d *decodeState) value(v reflect.Value) {
+ if !v.IsValid() {
+ _, rest, err := nextValue(d.data[d.off:], &d.nextscan)
+ if err != nil {
+ d.error(err)
+ }
+ d.off = len(d.data) - len(rest)
+
+ // d.scan thinks we're still at the beginning of the item.
+ // Feed in an empty string - the shortest, simplest value -
+ // so that it knows we got to the end of the value.
+ if d.scan.redo {
+ // rewind.
+ d.scan.redo = false
+ d.scan.step = stateBeginValue
+ }
+ d.scan.step(&d.scan, '"')
+ d.scan.step(&d.scan, '"')
+
+ n := len(d.scan.parseState)
+ if n > 0 && d.scan.parseState[n-1] == parseObjectKey {
+ // d.scan thinks we just read an object key; finish the object
+ d.scan.step(&d.scan, ':')
+ d.scan.step(&d.scan, '"')
+ d.scan.step(&d.scan, '"')
+ d.scan.step(&d.scan, '}')
+ }
+
+ return
+ }
+
+ switch op := d.scanWhile(scanSkipSpace); op {
+ default:
+ d.error(errPhase)
+
+ case scanBeginArray:
+ d.array(v)
+
+ case scanBeginObject:
+ d.object(v)
+
+ case scanBeginLiteral:
+ d.literal(v)
+
+ case scanBeginName:
+ d.name(v)
+ }
+}
+
+type unquotedValue struct{}
+
+// valueQuoted is like value but decodes a
+// quoted string literal or literal null into an interface value.
+// If it finds anything other than a quoted string literal or null,
+// valueQuoted returns unquotedValue{}.
+func (d *decodeState) valueQuoted() interface{} {
+ switch op := d.scanWhile(scanSkipSpace); op {
+ default:
+ d.error(errPhase)
+
+ case scanBeginArray:
+ d.array(reflect.Value{})
+
+ case scanBeginObject:
+ d.object(reflect.Value{})
+
+ case scanBeginName:
+ switch v := d.nameInterface().(type) {
+ case nil, string:
+ return v
+ }
+
+ case scanBeginLiteral:
+ switch v := d.literalInterface().(type) {
+ case nil, string:
+ return v
+ }
+ }
+ return unquotedValue{}
+}
+
+// indirect walks down v allocating pointers as needed,
+// until it gets to a non-pointer.
+// if it encounters an Unmarshaler, indirect stops and returns that.
+// if decodingNull is true, indirect stops at the last pointer so it can be set to nil.
+func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) {
+ // If v is a named type and is addressable,
+ // start with its address, so that if the type has pointer methods,
+ // we find them.
+ if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
+ v = v.Addr()
+ }
+ for {
+ // Load value from interface, but only if the result will be
+ // usefully addressable.
+ if v.Kind() == reflect.Interface && !v.IsNil() {
+ e := v.Elem()
+ if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) {
+ v = e
+ continue
+ }
+ }
+
+ if v.Kind() != reflect.Ptr {
+ break
+ }
+
+ if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() {
+ break
+ }
+ if v.IsNil() {
+ v.Set(reflect.New(v.Type().Elem()))
+ }
+ if v.Type().NumMethod() > 0 {
+ if u, ok := v.Interface().(Unmarshaler); ok {
+ return u, nil, v
+ }
+ if u, ok := v.Interface().(encoding.TextUnmarshaler); ok {
+ return nil, u, v
+ }
+ }
+ v = v.Elem()
+ }
+ return nil, nil, v
+}
+
+// array consumes an array from d.data[d.off-1:], decoding into the value v.
+// the first byte of the array ('[') has been read already.
+func (d *decodeState) array(v reflect.Value) {
+ // Check for unmarshaler.
+ u, ut, pv := d.indirect(v, false)
+ if u != nil {
+ d.off--
+ err := u.UnmarshalJSON(d.next())
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+ if ut != nil {
+ d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)})
+ d.off--
+ d.next()
+ return
+ }
+
+ v = pv
+
+ // Check type of target.
+ switch v.Kind() {
+ case reflect.Interface:
+ if v.NumMethod() == 0 {
+ // Decoding into nil interface? Switch to non-reflect code.
+ v.Set(reflect.ValueOf(d.arrayInterface()))
+ return
+ }
+ // Otherwise it's invalid.
+ fallthrough
+ default:
+ d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)})
+ d.off--
+ d.next()
+ return
+ case reflect.Array:
+ case reflect.Slice:
+ break
+ }
+
+ i := 0
+ for {
+ // Look ahead for ] - can only happen on first iteration.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndArray {
+ break
+ }
+
+ // Back up so d.value can have the byte we just read.
+ d.off--
+ d.scan.undo(op)
+
+ // Get element of array, growing if necessary.
+ if v.Kind() == reflect.Slice {
+ // Grow slice if necessary
+ if i >= v.Cap() {
+ newcap := v.Cap() + v.Cap()/2
+ if newcap < 4 {
+ newcap = 4
+ }
+ newv := reflect.MakeSlice(v.Type(), v.Len(), newcap)
+ reflect.Copy(newv, v)
+ v.Set(newv)
+ }
+ if i >= v.Len() {
+ v.SetLen(i + 1)
+ }
+ }
+
+ if i < v.Len() {
+ // Decode into element.
+ d.value(v.Index(i))
+ } else {
+ // Ran out of fixed array: skip.
+ d.value(reflect.Value{})
+ }
+ i++
+
+ // Next token must be , or ].
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndArray {
+ break
+ }
+ if op != scanArrayValue {
+ d.error(errPhase)
+ }
+ }
+
+ if i < v.Len() {
+ if v.Kind() == reflect.Array {
+ // Array. Zero the rest.
+ z := reflect.Zero(v.Type().Elem())
+ for ; i < v.Len(); i++ {
+ v.Index(i).Set(z)
+ }
+ } else {
+ v.SetLen(i)
+ }
+ }
+ if i == 0 && v.Kind() == reflect.Slice {
+ v.Set(reflect.MakeSlice(v.Type(), 0, 0))
+ }
+}
+
+var nullLiteral = []byte("null")
+var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem()
+
+// object consumes an object from d.data[d.off-1:], decoding into the value v.
+// the first byte ('{') of the object has been read already.
+func (d *decodeState) object(v reflect.Value) {
+ // Check for unmarshaler.
+ u, ut, pv := d.indirect(v, false)
+ if d.storeKeyed(pv) {
+ return
+ }
+ if u != nil {
+ d.off--
+ err := u.UnmarshalJSON(d.next())
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+ if ut != nil {
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over { } in input
+ return
+ }
+ v = pv
+
+ // Decoding into nil interface? Switch to non-reflect code.
+ if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
+ v.Set(reflect.ValueOf(d.objectInterface()))
+ return
+ }
+
+ // Check type of target:
+ // struct or
+ // map[string]T or map[encoding.TextUnmarshaler]T
+ switch v.Kind() {
+ case reflect.Map:
+ // Map key must either have string kind or be an encoding.TextUnmarshaler.
+ t := v.Type()
+ if t.Key().Kind() != reflect.String &&
+ !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) {
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over { } in input
+ return
+ }
+ if v.IsNil() {
+ v.Set(reflect.MakeMap(t))
+ }
+ case reflect.Struct:
+
+ default:
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over { } in input
+ return
+ }
+
+ var mapElem reflect.Value
+
+ empty := true
+ for {
+ // Read opening " of string key or closing }.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndObject {
+ if !empty && !d.ext.trailingCommas {
+ d.syntaxError("beginning of object key string")
+ }
+ break
+ }
+ empty = false
+ if op == scanBeginName {
+ if !d.ext.unquotedKeys {
+ d.syntaxError("beginning of object key string")
+ }
+ } else if op != scanBeginLiteral {
+ d.error(errPhase)
+ }
+ unquotedKey := op == scanBeginName
+
+ // Read key.
+ start := d.off - 1
+ op = d.scanWhile(scanContinue)
+ item := d.data[start : d.off-1]
+ var key []byte
+ if unquotedKey {
+ key = item
+ // TODO Fix code below to quote item when necessary.
+ } else {
+ var ok bool
+ key, ok = unquoteBytes(item)
+ if !ok {
+ d.error(errPhase)
+ }
+ }
+
+ // Figure out field corresponding to key.
+ var subv reflect.Value
+ destring := false // whether the value is wrapped in a string to be decoded first
+
+ if v.Kind() == reflect.Map {
+ elemType := v.Type().Elem()
+ if !mapElem.IsValid() {
+ mapElem = reflect.New(elemType).Elem()
+ } else {
+ mapElem.Set(reflect.Zero(elemType))
+ }
+ subv = mapElem
+ } else {
+ var f *field
+ fields := cachedTypeFields(v.Type())
+ for i := range fields {
+ ff := &fields[i]
+ if bytes.Equal(ff.nameBytes, key) {
+ f = ff
+ break
+ }
+ if f == nil && ff.equalFold(ff.nameBytes, key) {
+ f = ff
+ }
+ }
+ if f != nil {
+ subv = v
+ destring = f.quoted
+ for _, i := range f.index {
+ if subv.Kind() == reflect.Ptr {
+ if subv.IsNil() {
+ subv.Set(reflect.New(subv.Type().Elem()))
+ }
+ subv = subv.Elem()
+ }
+ subv = subv.Field(i)
+ }
+ }
+ }
+
+ // Read : before value.
+ if op == scanSkipSpace {
+ op = d.scanWhile(scanSkipSpace)
+ }
+ if op != scanObjectKey {
+ d.error(errPhase)
+ }
+
+ // Read value.
+ if destring {
+ switch qv := d.valueQuoted().(type) {
+ case nil:
+ d.literalStore(nullLiteral, subv, false)
+ case string:
+ d.literalStore([]byte(qv), subv, true)
+ default:
+ d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type()))
+ }
+ } else {
+ d.value(subv)
+ }
+
+ // Write value back to map;
+ // if using struct, subv points into struct already.
+ if v.Kind() == reflect.Map {
+ kt := v.Type().Key()
+ var kv reflect.Value
+ switch {
+ case kt.Kind() == reflect.String:
+ kv = reflect.ValueOf(key).Convert(v.Type().Key())
+ case reflect.PtrTo(kt).Implements(textUnmarshalerType):
+ kv = reflect.New(v.Type().Key())
+ d.literalStore(item, kv, true)
+ kv = kv.Elem()
+ default:
+ panic("json: Unexpected key type") // should never occur
+ }
+ v.SetMapIndex(kv, subv)
+ }
+
+ // Next token must be , or }.
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndObject {
+ break
+ }
+ if op != scanObjectValue {
+ d.error(errPhase)
+ }
+ }
+}
+
+// isNull returns whether there's a null literal at the provided offset.
+func (d *decodeState) isNull(off int) bool {
+ if off+4 >= len(d.data) || d.data[off] != 'n' || d.data[off+1] != 'u' || d.data[off+2] != 'l' || d.data[off+3] != 'l' {
+ return false
+ }
+ d.nextscan.reset()
+ for i, c := range d.data[off:] {
+ if i > 4 {
+ return false
+ }
+ switch d.nextscan.step(&d.nextscan, c) {
+ case scanContinue, scanBeginName:
+ continue
+ }
+ break
+ }
+ return true
+}
+
+// name consumes a const or function from d.data[d.off-1:], decoding into the value v.
+// the first byte of the function name has been read already.
+func (d *decodeState) name(v reflect.Value) {
+ if d.isNull(d.off - 1) {
+ d.literal(v)
+ return
+ }
+
+ // Check for unmarshaler.
+ u, ut, pv := d.indirect(v, false)
+ if d.storeKeyed(pv) {
+ return
+ }
+ if u != nil {
+ d.off--
+ err := u.UnmarshalJSON(d.next())
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+ if ut != nil {
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over function in input
+ return
+ }
+ v = pv
+
+ // Decoding into nil interface? Switch to non-reflect code.
+ if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
+ out := d.nameInterface()
+ if out == nil {
+ v.Set(reflect.Zero(v.Type()))
+ } else {
+ v.Set(reflect.ValueOf(out))
+ }
+ return
+ }
+
+ nameStart := d.off - 1
+
+ op := d.scanWhile(scanContinue)
+
+ name := d.data[nameStart : d.off-1]
+ if op != scanParam {
+ // Back up so the byte just read is consumed next.
+ d.off--
+ d.scan.undo(op)
+ if l, ok := d.convertLiteral(name); ok {
+ d.storeValue(v, l)
+ return
+ }
+ d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)})
+ }
+
+ funcName := string(name)
+ funcData := d.ext.funcs[funcName]
+ if funcData.key == "" {
+ d.error(fmt.Errorf("json: unknown function %q", funcName))
+ }
+
+ // Check type of target:
+ // struct or
+ // map[string]T or map[encoding.TextUnmarshaler]T
+ switch v.Kind() {
+ case reflect.Map:
+ // Map key must either have string kind or be an encoding.TextUnmarshaler.
+ t := v.Type()
+ if t.Key().Kind() != reflect.String &&
+ !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) {
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over { } in input
+ return
+ }
+ if v.IsNil() {
+ v.Set(reflect.MakeMap(t))
+ }
+ case reflect.Struct:
+
+ default:
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ d.off--
+ d.next() // skip over { } in input
+ return
+ }
+
+ // TODO Fix case of func field as map.
+ //topv := v
+
+ // Figure out field corresponding to function.
+ key := []byte(funcData.key)
+ if v.Kind() == reflect.Map {
+ elemType := v.Type().Elem()
+ v = reflect.New(elemType).Elem()
+ } else {
+ var f *field
+ fields := cachedTypeFields(v.Type())
+ for i := range fields {
+ ff := &fields[i]
+ if bytes.Equal(ff.nameBytes, key) {
+ f = ff
+ break
+ }
+ if f == nil && ff.equalFold(ff.nameBytes, key) {
+ f = ff
+ }
+ }
+ if f != nil {
+ for _, i := range f.index {
+ if v.Kind() == reflect.Ptr {
+ if v.IsNil() {
+ v.Set(reflect.New(v.Type().Elem()))
+ }
+ v = v.Elem()
+ }
+ v = v.Field(i)
+ }
+ if v.Kind() == reflect.Ptr {
+ if v.IsNil() {
+ v.Set(reflect.New(v.Type().Elem()))
+ }
+ v = v.Elem()
+ }
+ }
+ }
+
+ // Check for unmarshaler on func field itself.
+ u, _, _ = d.indirect(v, false)
+ if u != nil {
+ d.off = nameStart
+ err := u.UnmarshalJSON(d.next())
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+
+ var mapElem reflect.Value
+
+ // Parse function arguments.
+ for i := 0; ; i++ {
+ // closing ) - can only happen on first iteration.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndParams {
+ break
+ }
+
+ // Back up so d.value can have the byte we just read.
+ d.off--
+ d.scan.undo(op)
+
+ if i >= len(funcData.args) {
+ d.error(fmt.Errorf("json: too many arguments for function %s", funcName))
+ }
+ key := []byte(funcData.args[i])
+
+ // Figure out field corresponding to key.
+ var subv reflect.Value
+ destring := false // whether the value is wrapped in a string to be decoded first
+
+ if v.Kind() == reflect.Map {
+ elemType := v.Type().Elem()
+ if !mapElem.IsValid() {
+ mapElem = reflect.New(elemType).Elem()
+ } else {
+ mapElem.Set(reflect.Zero(elemType))
+ }
+ subv = mapElem
+ } else {
+ var f *field
+ fields := cachedTypeFields(v.Type())
+ for i := range fields {
+ ff := &fields[i]
+ if bytes.Equal(ff.nameBytes, key) {
+ f = ff
+ break
+ }
+ if f == nil && ff.equalFold(ff.nameBytes, key) {
+ f = ff
+ }
+ }
+ if f != nil {
+ subv = v
+ destring = f.quoted
+ for _, i := range f.index {
+ if subv.Kind() == reflect.Ptr {
+ if subv.IsNil() {
+ subv.Set(reflect.New(subv.Type().Elem()))
+ }
+ subv = subv.Elem()
+ }
+ subv = subv.Field(i)
+ }
+ }
+ }
+
+ // Read value.
+ if destring {
+ switch qv := d.valueQuoted().(type) {
+ case nil:
+ d.literalStore(nullLiteral, subv, false)
+ case string:
+ d.literalStore([]byte(qv), subv, true)
+ default:
+ d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type()))
+ }
+ } else {
+ d.value(subv)
+ }
+
+ // Write value back to map;
+ // if using struct, subv points into struct already.
+ if v.Kind() == reflect.Map {
+ kt := v.Type().Key()
+ var kv reflect.Value
+ switch {
+ case kt.Kind() == reflect.String:
+ kv = reflect.ValueOf(key).Convert(v.Type().Key())
+ case reflect.PtrTo(kt).Implements(textUnmarshalerType):
+ kv = reflect.New(v.Type().Key())
+ d.literalStore(key, kv, true)
+ kv = kv.Elem()
+ default:
+ panic("json: Unexpected key type") // should never occur
+ }
+ v.SetMapIndex(kv, subv)
+ }
+
+ // Next token must be , or ).
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndParams {
+ break
+ }
+ if op != scanParam {
+ d.error(errPhase)
+ }
+ }
+}
+
+// keyed attempts to decode an object or function using a keyed doc extension,
+// and returns the value and true on success, or nil and false otherwise.
+func (d *decodeState) keyed() (interface{}, bool) {
+ if len(d.ext.keyed) == 0 {
+ return nil, false
+ }
+
+ unquote := false
+
+ // Look-ahead first key to check for a keyed document extension.
+ d.nextscan.reset()
+ var start, end int
+ for i, c := range d.data[d.off-1:] {
+ switch op := d.nextscan.step(&d.nextscan, c); op {
+ case scanSkipSpace, scanContinue, scanBeginObject:
+ continue
+ case scanBeginLiteral, scanBeginName:
+ unquote = op == scanBeginLiteral
+ start = i
+ continue
+ }
+ end = i
+ break
+ }
+
+ name := bytes.Trim(d.data[d.off-1+start:d.off-1+end], " \n\t")
+
+ var key []byte
+ var ok bool
+ if unquote {
+ key, ok = unquoteBytes(name)
+ if !ok {
+ d.error(errPhase)
+ }
+ } else {
+ funcData, ok := d.ext.funcs[string(name)]
+ if !ok {
+ return nil, false
+ }
+ key = []byte(funcData.key)
+ }
+
+ decode, ok := d.ext.keyed[string(key)]
+ if !ok {
+ return nil, false
+ }
+
+ d.off--
+ out, err := decode(d.next())
+ if err != nil {
+ d.error(err)
+ }
+ return out, true
+}
+
+func (d *decodeState) storeKeyed(v reflect.Value) bool {
+ keyed, ok := d.keyed()
+ if !ok {
+ return false
+ }
+ d.storeValue(v, keyed)
+ return true
+}
+
+var (
+ trueBytes = []byte("true")
+ falseBytes = []byte("false")
+ nullBytes = []byte("null")
+)
+
+func (d *decodeState) storeValue(v reflect.Value, from interface{}) {
+ switch from {
+ case nil:
+ d.literalStore(nullBytes, v, false)
+ return
+ case true:
+ d.literalStore(trueBytes, v, false)
+ return
+ case false:
+ d.literalStore(falseBytes, v, false)
+ return
+ }
+ fromv := reflect.ValueOf(from)
+ for fromv.Kind() == reflect.Ptr && !fromv.IsNil() {
+ fromv = fromv.Elem()
+ }
+ fromt := fromv.Type()
+ for v.Kind() == reflect.Ptr && !v.IsNil() {
+ v = v.Elem()
+ }
+ vt := v.Type()
+ if fromt.AssignableTo(vt) {
+ v.Set(fromv)
+ } else if fromt.ConvertibleTo(vt) {
+ v.Set(fromv.Convert(vt))
+ } else {
+ d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)})
+ }
+}
+
+func (d *decodeState) convertLiteral(name []byte) (interface{}, bool) {
+ if len(name) == 0 {
+ return nil, false
+ }
+ switch name[0] {
+ case 't':
+ if bytes.Equal(name, trueBytes) {
+ return true, true
+ }
+ case 'f':
+ if bytes.Equal(name, falseBytes) {
+ return false, true
+ }
+ case 'n':
+ if bytes.Equal(name, nullBytes) {
+ return nil, true
+ }
+ }
+ if l, ok := d.ext.consts[string(name)]; ok {
+ return l, true
+ }
+ return nil, false
+}
+
+// literal consumes a literal from d.data[d.off-1:], decoding into the value v.
+// The first byte of the literal has been read already
+// (that's how the caller knows it's a literal).
+func (d *decodeState) literal(v reflect.Value) {
+ // All bytes inside literal return scanContinue op code.
+ start := d.off - 1
+ op := d.scanWhile(scanContinue)
+
+ // Scan read one byte too far; back up.
+ d.off--
+ d.scan.undo(op)
+
+ d.literalStore(d.data[start:d.off], v, false)
+}
+
+// convertNumber converts the number literal s to a float64 or a Number
+// depending on the setting of d.useNumber.
+func (d *decodeState) convertNumber(s string) (interface{}, error) {
+ if d.useNumber {
+ return Number(s), nil
+ }
+ f, err := strconv.ParseFloat(s, 64)
+ if err != nil {
+ return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)}
+ }
+ return f, nil
+}
+
+var numberType = reflect.TypeOf(Number(""))
+
+// literalStore decodes a literal stored in item into v.
+//
+// fromQuoted indicates whether this literal came from unwrapping a
+// string from the ",string" struct tag option. this is used only to
+// produce more helpful error messages.
+func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) {
+ // Check for unmarshaler.
+ if len(item) == 0 {
+ //Empty string given
+ d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ return
+ }
+ wantptr := item[0] == 'n' // null
+ u, ut, pv := d.indirect(v, wantptr)
+ if u != nil {
+ err := u.UnmarshalJSON(item)
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+ if ut != nil {
+ if item[0] != '"' {
+ if fromQuoted {
+ d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)})
+ }
+ return
+ }
+ s, ok := unquoteBytes(item)
+ if !ok {
+ if fromQuoted {
+ d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.error(errPhase)
+ }
+ }
+ err := ut.UnmarshalText(s)
+ if err != nil {
+ d.error(err)
+ }
+ return
+ }
+
+ v = pv
+
+ switch c := item[0]; c {
+ case 'n': // null
+ switch v.Kind() {
+ case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice:
+ v.Set(reflect.Zero(v.Type()))
+ // otherwise, ignore null for primitives/string
+ }
+ case 't', 'f': // true, false
+ value := c == 't'
+ switch v.Kind() {
+ default:
+ if fromQuoted {
+ d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)})
+ }
+ case reflect.Bool:
+ v.SetBool(value)
+ case reflect.Interface:
+ if v.NumMethod() == 0 {
+ v.Set(reflect.ValueOf(value))
+ } else {
+ d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)})
+ }
+ }
+
+ case '"': // string
+ s, ok := unquoteBytes(item)
+ if !ok {
+ if fromQuoted {
+ d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.error(errPhase)
+ }
+ }
+ switch v.Kind() {
+ default:
+ d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)})
+ case reflect.Slice:
+ if v.Type().Elem().Kind() != reflect.Uint8 {
+ d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)})
+ break
+ }
+ b := make([]byte, base64.StdEncoding.DecodedLen(len(s)))
+ n, err := base64.StdEncoding.Decode(b, s)
+ if err != nil {
+ d.saveError(err)
+ break
+ }
+ v.SetBytes(b[:n])
+ case reflect.String:
+ v.SetString(string(s))
+ case reflect.Interface:
+ if v.NumMethod() == 0 {
+ v.Set(reflect.ValueOf(string(s)))
+ } else {
+ d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)})
+ }
+ }
+
+ default: // number
+ if c != '-' && (c < '0' || c > '9') {
+ if fromQuoted {
+ d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.error(errPhase)
+ }
+ }
+ s := string(item)
+ switch v.Kind() {
+ default:
+ if v.Kind() == reflect.String && v.Type() == numberType {
+ v.SetString(s)
+ if !isValidNumber(s) {
+ d.error(fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item))
+ }
+ break
+ }
+ if fromQuoted {
+ d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()))
+ } else {
+ d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)})
+ }
+ case reflect.Interface:
+ n, err := d.convertNumber(s)
+ if err != nil {
+ d.saveError(err)
+ break
+ }
+ if v.NumMethod() != 0 {
+ d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)})
+ break
+ }
+ v.Set(reflect.ValueOf(n))
+
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ n, err := strconv.ParseInt(s, 10, 64)
+ if err != nil || v.OverflowInt(n) {
+ d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)})
+ break
+ }
+ v.SetInt(n)
+
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ n, err := strconv.ParseUint(s, 10, 64)
+ if err != nil || v.OverflowUint(n) {
+ d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)})
+ break
+ }
+ v.SetUint(n)
+
+ case reflect.Float32, reflect.Float64:
+ n, err := strconv.ParseFloat(s, v.Type().Bits())
+ if err != nil || v.OverflowFloat(n) {
+ d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)})
+ break
+ }
+ v.SetFloat(n)
+ }
+ }
+}
+
+// The xxxInterface routines build up a value to be stored
+// in an empty interface. They are not strictly necessary,
+// but they avoid the weight of reflection in this common case.
+
+// valueInterface is like value but returns interface{}
+func (d *decodeState) valueInterface() interface{} {
+ switch d.scanWhile(scanSkipSpace) {
+ default:
+ d.error(errPhase)
+ panic("unreachable")
+ case scanBeginArray:
+ return d.arrayInterface()
+ case scanBeginObject:
+ return d.objectInterface()
+ case scanBeginLiteral:
+ return d.literalInterface()
+ case scanBeginName:
+ return d.nameInterface()
+ }
+}
+
+func (d *decodeState) syntaxError(expected string) {
+ msg := fmt.Sprintf("invalid character '%c' looking for %s", d.data[d.off-1], expected)
+ d.error(&SyntaxError{msg, int64(d.off)})
+}
+
+// arrayInterface is like array but returns []interface{}.
+func (d *decodeState) arrayInterface() []interface{} {
+ var v = make([]interface{}, 0)
+ for {
+ // Look ahead for ] - can only happen on first iteration.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndArray {
+ if len(v) > 0 && !d.ext.trailingCommas {
+ d.syntaxError("beginning of value")
+ }
+ break
+ }
+
+ // Back up so d.value can have the byte we just read.
+ d.off--
+ d.scan.undo(op)
+
+ v = append(v, d.valueInterface())
+
+ // Next token must be , or ].
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndArray {
+ break
+ }
+ if op != scanArrayValue {
+ d.error(errPhase)
+ }
+ }
+ return v
+}
+
+// objectInterface is like object but returns map[string]interface{}.
+func (d *decodeState) objectInterface() interface{} {
+ v, ok := d.keyed()
+ if ok {
+ return v
+ }
+
+ m := make(map[string]interface{})
+ for {
+ // Read opening " of string key or closing }.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndObject {
+ if len(m) > 0 && !d.ext.trailingCommas {
+ d.syntaxError("beginning of object key string")
+ }
+ break
+ }
+ if op == scanBeginName {
+ if !d.ext.unquotedKeys {
+ d.syntaxError("beginning of object key string")
+ }
+ } else if op != scanBeginLiteral {
+ d.error(errPhase)
+ }
+ unquotedKey := op == scanBeginName
+
+ // Read string key.
+ start := d.off - 1
+ op = d.scanWhile(scanContinue)
+ item := d.data[start : d.off-1]
+ var key string
+ if unquotedKey {
+ key = string(item)
+ } else {
+ var ok bool
+ key, ok = unquote(item)
+ if !ok {
+ d.error(errPhase)
+ }
+ }
+
+ // Read : before value.
+ if op == scanSkipSpace {
+ op = d.scanWhile(scanSkipSpace)
+ }
+ if op != scanObjectKey {
+ d.error(errPhase)
+ }
+
+ // Read value.
+ m[key] = d.valueInterface()
+
+ // Next token must be , or }.
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndObject {
+ break
+ }
+ if op != scanObjectValue {
+ d.error(errPhase)
+ }
+ }
+ return m
+}
+
+// literalInterface is like literal but returns an interface value.
+func (d *decodeState) literalInterface() interface{} {
+ // All bytes inside literal return scanContinue op code.
+ start := d.off - 1
+ op := d.scanWhile(scanContinue)
+
+ // Scan read one byte too far; back up.
+ d.off--
+ d.scan.undo(op)
+ item := d.data[start:d.off]
+
+ switch c := item[0]; c {
+ case 'n': // null
+ return nil
+
+ case 't', 'f': // true, false
+ return c == 't'
+
+ case '"': // string
+ s, ok := unquote(item)
+ if !ok {
+ d.error(errPhase)
+ }
+ return s
+
+ default: // number
+ if c != '-' && (c < '0' || c > '9') {
+ d.error(errPhase)
+ }
+ n, err := d.convertNumber(string(item))
+ if err != nil {
+ d.saveError(err)
+ }
+ return n
+ }
+}
+
+// nameInterface is like function but returns map[string]interface{}.
+func (d *decodeState) nameInterface() interface{} {
+ v, ok := d.keyed()
+ if ok {
+ return v
+ }
+
+ nameStart := d.off - 1
+
+ op := d.scanWhile(scanContinue)
+
+ name := d.data[nameStart : d.off-1]
+ if op != scanParam {
+ // Back up so the byte just read is consumed next.
+ d.off--
+ d.scan.undo(op)
+ if l, ok := d.convertLiteral(name); ok {
+ return l
+ }
+ d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)})
+ }
+
+ funcName := string(name)
+ funcData := d.ext.funcs[funcName]
+ if funcData.key == "" {
+ d.error(fmt.Errorf("json: unknown function %q", funcName))
+ }
+
+ m := make(map[string]interface{})
+ for i := 0; ; i++ {
+ // Look ahead for ) - can only happen on first iteration.
+ op := d.scanWhile(scanSkipSpace)
+ if op == scanEndParams {
+ break
+ }
+
+ // Back up so d.value can have the byte we just read.
+ d.off--
+ d.scan.undo(op)
+
+ if i >= len(funcData.args) {
+ d.error(fmt.Errorf("json: too many arguments for function %s", funcName))
+ }
+ m[funcData.args[i]] = d.valueInterface()
+
+ // Next token must be , or ).
+ op = d.scanWhile(scanSkipSpace)
+ if op == scanEndParams {
+ break
+ }
+ if op != scanParam {
+ d.error(errPhase)
+ }
+ }
+ return map[string]interface{}{funcData.key: m}
+}
+
+// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
+// or it returns -1.
+func getu4(s []byte) rune {
+ if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
+ return -1
+ }
+ r, err := strconv.ParseUint(string(s[2:6]), 16, 64)
+ if err != nil {
+ return -1
+ }
+ return rune(r)
+}
+
+// unquote converts a quoted JSON string literal s into an actual string t.
+// The rules are different than for Go, so cannot use strconv.Unquote.
+func unquote(s []byte) (t string, ok bool) {
+ s, ok = unquoteBytes(s)
+ t = string(s)
+ return
+}
+
+func unquoteBytes(s []byte) (t []byte, ok bool) {
+ if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
+ return
+ }
+ s = s[1 : len(s)-1]
+
+ // Check for unusual characters. If there are none,
+ // then no unquoting is needed, so return a slice of the
+ // original bytes.
+ r := 0
+ for r < len(s) {
+ c := s[r]
+ if c == '\\' || c == '"' || c < ' ' {
+ break
+ }
+ if c < utf8.RuneSelf {
+ r++
+ continue
+ }
+ rr, size := utf8.DecodeRune(s[r:])
+ if rr == utf8.RuneError && size == 1 {
+ break
+ }
+ r += size
+ }
+ if r == len(s) {
+ return s, true
+ }
+
+ b := make([]byte, len(s)+2*utf8.UTFMax)
+ w := copy(b, s[0:r])
+ for r < len(s) {
+ // Out of room? Can only happen if s is full of
+ // malformed UTF-8 and we're replacing each
+ // byte with RuneError.
+ if w >= len(b)-2*utf8.UTFMax {
+ nb := make([]byte, (len(b)+utf8.UTFMax)*2)
+ copy(nb, b[0:w])
+ b = nb
+ }
+ switch c := s[r]; {
+ case c == '\\':
+ r++
+ if r >= len(s) {
+ return
+ }
+ switch s[r] {
+ default:
+ return
+ case '"', '\\', '/', '\'':
+ b[w] = s[r]
+ r++
+ w++
+ case 'b':
+ b[w] = '\b'
+ r++
+ w++
+ case 'f':
+ b[w] = '\f'
+ r++
+ w++
+ case 'n':
+ b[w] = '\n'
+ r++
+ w++
+ case 'r':
+ b[w] = '\r'
+ r++
+ w++
+ case 't':
+ b[w] = '\t'
+ r++
+ w++
+ case 'u':
+ r--
+ rr := getu4(s[r:])
+ if rr < 0 {
+ return
+ }
+ r += 6
+ if utf16.IsSurrogate(rr) {
+ rr1 := getu4(s[r:])
+ if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
+ // A valid pair; consume.
+ r += 6
+ w += utf8.EncodeRune(b[w:], dec)
+ break
+ }
+ // Invalid surrogate; fall back to replacement rune.
+ rr = unicode.ReplacementChar
+ }
+ w += utf8.EncodeRune(b[w:], rr)
+ }
+
+ // Quote, control characters are invalid.
+ case c == '"', c < ' ':
+ return
+
+ // ASCII
+ case c < utf8.RuneSelf:
+ b[w] = c
+ r++
+ w++
+
+ // Coerce to well-formed UTF-8.
+ default:
+ rr, size := utf8.DecodeRune(s[r:])
+ r += size
+ w += utf8.EncodeRune(b[w:], rr)
+ }
+ }
+ return b[0:w], true
+}
diff --git a/backend/vendor/github.com/globalsign/mgo/internal/json/encode.go b/backend/vendor/github.com/globalsign/mgo/internal/json/encode.go
new file mode 100644
index 00000000..e4b8f864
--- /dev/null
+++ b/backend/vendor/github.com/globalsign/mgo/internal/json/encode.go
@@ -0,0 +1,1260 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package json implements encoding and decoding of JSON as defined in
+// RFC 4627. The mapping between JSON and Go values is described
+// in the documentation for the Marshal and Unmarshal functions.
+//
+// See "JSON and Go" for an introduction to this package:
+// https://golang.org/doc/articles/json_and_go.html
+package json
+
+import (
+ "bytes"
+ "encoding"
+ "encoding/base64"
+ "fmt"
+ "math"
+ "reflect"
+ "runtime"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "unicode"
+ "unicode/utf8"
+)
+
+// Marshal returns the JSON encoding of v.
+//
+// Marshal traverses the value v recursively.
+// If an encountered value implements the Marshaler interface
+// and is not a nil pointer, Marshal calls its MarshalJSON method
+// to produce JSON. If no MarshalJSON method is present but the
+// value implements encoding.TextMarshaler instead, Marshal calls
+// its MarshalText method.
+// The nil pointer exception is not strictly necessary
+// but mimics a similar, necessary exception in the behavior of
+// UnmarshalJSON.
+//
+// Otherwise, Marshal uses the following type-dependent default encodings:
+//
+// Boolean values encode as JSON booleans.
+//
+// Floating point, integer, and Number values encode as JSON numbers.
+//
+// String values encode as JSON strings coerced to valid UTF-8,
+// replacing invalid bytes with the Unicode replacement rune.
+// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e"
+// to keep some browsers from misinterpreting JSON output as HTML.
+// Ampersand "&" is also escaped to "\u0026" for the same reason.
+// This escaping can be disabled using an Encoder with DisableHTMLEscaping.
+//
+// Array and slice values encode as JSON arrays, except that
+// []byte encodes as a base64-encoded string, and a nil slice
+// encodes as the null JSON value.
+//
+// Struct values encode as JSON objects. Each exported struct field
+// becomes a member of the object unless
+// - the field's tag is "-", or
+// - the field is empty and its tag specifies the "omitempty" option.
+// The empty values are false, 0, any
+// nil pointer or interface value, and any array, slice, map, or string of
+// length zero. The object's default key string is the struct field name
+// but can be specified in the struct field's tag value. The "json" key in
+// the struct field's tag value is the key name, followed by an optional comma
+// and options. Examples:
+//
+// // Field is ignored by this package.
+// Field int `json:"-"`
+//
+// // Field appears in JSON as key "myName".
+// Field int `json:"myName"`
+//
+// // Field appears in JSON as key "myName" and
+// // the field is omitted from the object if its value is empty,
+// // as defined above.
+// Field int `json:"myName,omitempty"`
+//
+// // Field appears in JSON as key "Field" (the default), but
+// // the field is skipped if empty.
+// // Note the leading comma.
+// Field int `json:",omitempty"`
+//
+// The "string" option signals that a field is stored as JSON inside a
+// JSON-encoded string. It applies only to fields of string, floating point,
+// integer, or boolean types. This extra level of encoding is sometimes used
+// when communicating with JavaScript programs:
+//
+// Int64String int64 `json:",string"`
+//
+// The key name will be used if it's a non-empty string consisting of
+// only Unicode letters, digits, dollar signs, percent signs, hyphens,
+// underscores and slashes.
+//
+// Anonymous struct fields are usually marshaled as if their inner exported fields
+// were fields in the outer struct, subject to the usual Go visibility rules amended
+// as described in the next paragraph.
+// An anonymous struct field with a name given in its JSON tag is treated as
+// having that name, rather than being anonymous.
+// An anonymous struct field of interface type is treated the same as having
+// that type as its name, rather than being anonymous.
+//
+// The Go visibility rules for struct fields are amended for JSON when
+// deciding which field to marshal or unmarshal. If there are
+// multiple fields at the same level, and that level is the least
+// nested (and would therefore be the nesting level selected by the
+// usual Go rules), the following extra rules apply:
+//
+// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered,
+// even if there are multiple untagged fields that would otherwise conflict.
+// 2) If there is exactly one field (tagged or not according to the first rule), that is selected.
+// 3) Otherwise there are multiple fields, and all are ignored; no error occurs.
+//
+// Handling of anonymous struct fields is new in Go 1.1.
+// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of
+// an anonymous struct field in both current and earlier versions, give the field
+// a JSON tag of "-".
+//
+// Map values encode as JSON objects. The map's key type must either be a string
+// or implement encoding.TextMarshaler. The map keys are used as JSON object
+// keys, subject to the UTF-8 coercion described for string values above.
+//
+// Pointer values encode as the value pointed to.
+// A nil pointer encodes as the null JSON value.
+//
+// Interface values encode as the value contained in the interface.
+// A nil interface value encodes as the null JSON value.
+//
+// Channel, complex, and function values cannot be encoded in JSON.
+// Attempting to encode such a value causes Marshal to return
+// an UnsupportedTypeError.
+//
+// JSON cannot represent cyclic data structures and Marshal does not
+// handle them. Passing cyclic structures to Marshal will result in
+// an infinite recursion.
+//
+func Marshal(v interface{}) ([]byte, error) {
+ e := &encodeState{}
+ err := e.marshal(v, encOpts{escapeHTML: true})
+ if err != nil {
+ return nil, err
+ }
+ return e.Bytes(), nil
+}
+
+// MarshalIndent is like Marshal but applies Indent to format the output.
+func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
+ b, err := Marshal(v)
+ if err != nil {
+ return nil, err
+ }
+ var buf bytes.Buffer
+ err = Indent(&buf, b, prefix, indent)
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
+
+// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029
+// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029
+// so that the JSON will be safe to embed inside HTML