mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/ext-mongo (#177)

Browse Source 
* externalize mongo wrapper

* remove mongo env from drone

* export mongo test for coverage report
This commit is contained in:
Umputun
2018-07-22 22:46:25 -04:00
committed by GitHub
parent fdf161850b
commit dbd1d4069f
27 changed files with 311 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+2 -4
View File
@@ -16,13 +16,12 @@ pipeline:
docker_master:
image: plugins/docker
repo: umputun/remark42
secrets: [ docker_username, docker_password, mongo_remark_test ]
secrets: [ docker_username, docker_password]
build_args:
- DRONE=${DRONE}
- DRONE_TAG=${DRONE_TAG}
- DRONE_COMMIT=${DRONE_COMMIT}
- DRONE_BRANCH=${DRONE_BRANCH}
build_args_from_env: [mongo_remark_test]
tags:
- ${DRONE_COMMIT_BRANCH/\//-}
when:
@@ -46,12 +45,11 @@ pipeline:
docker_branch:
image: plugins/docker
repo: umputun/remark42
secrets: [ docker_username, docker_password, mongo_remark_test ]
secrets: [ docker_username, docker_password ]
build_args:
- DRONE=${DRONE}
- DRONE_COMMIT=${DRONE_COMMIT}
- DRONE_BRANCH=${DRONE_BRANCH}
build_args_from_env: [mongo_remark_test]
tags:
- ${DRONE_COMMIT_BRANCH/\//-}
when:
.travis.yml
+1 -1
View File
@@ -16,5 +16,5 @@ script:
--build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
--build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
--build-arg TRAVIS_TAG=$TRAVIS_TAG
--build-arg MONGO_REMARK_TEST=$MONGO_REMARK_TEST
--build-arg MONGO_TEST=$MONGO_REMARK_TEST
.
Dockerfile
+5 -4
View File
@@ -19,23 +19,23 @@ ARG DRONE_BRANCH
ARG DRONE_PULL_REQUEST
ARG SKIP_BACKEND_TEST
ARG MONGO_REMARK_TEST
ARG MONGO_TEST
WORKDIR /go/src/github.com/umputun/remark/backend
ADD backend /go/src/github.com/umputun/remark/backend
# run tests
RUN \
if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
cd app && \
if [ -z "$SKIP_BACKEND_TEST" ] ; then go test ./... ; \
else echo "skip backend test" ; fi
RUN echo "mongo=${MONGO_REMARK_TEST}" >> /etc/hosts
RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
# linters
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
if [ -f .mongo ] ; then export MONGO_REMARK_TEST=$(cat .mongo) ; fi && \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
--enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
--enable=deadcode --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \
@@ -43,6 +43,7 @@ RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
# coverage report
RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
mkdir -p target && /script/coverage.sh ; \
else echo "skip backend coverage" ; fi
backend/Gopkg.lock
Generated
+7 -1
View File
@@ -81,6 +81,12 @@
revision = "9f855fadd4b8cde7773f9ef51f6b2705af239519"
version = "v1.0.0"
[[projects]]
name = "github.com/go-pkgz/mongo"
packages = ["."]
revision = "9a09a574c336c6ae2338a65bbebed2baab2a713c"
version = "v1.0.0"
[[projects]]
name = "github.com/golang/protobuf"
packages = ["proto"]
@@ -250,6 +256,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "b6dd6c99dd3a4f5135f8b869063a2b24044494b2ed77442bfd50a45d096e414a"
inputs-digest = "e933250b2582d1ff7d2b165c4a8a934762677a2ebaaea743fb3c485868c6d793"
solver-name = "gps-cdcl"
solver-version = 1
backend/app/main.go
+2 -3
View File
@@ -11,9 +11,8 @@ import (
"syscall"
"time"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/coreos/bbolt"
"github.com/go-pkgz/mongo"
"github.com/hashicorp/logutils"
"github.com/jessevdk/go-flags"
"github.com/pkg/errors"
@@ -23,8 +22,8 @@ import (
"github.com/umputun/remark/backend/app/rest/auth"
"github.com/umputun/remark/backend/app/rest/cache"
"github.com/umputun/remark/backend/app/rest/proxy"
"github.com/umputun/remark/backend/app/store/avatar"
"github.com/umputun/remark/backend/app/store/engine"
"github.com/umputun/remark/backend/app/store/engine/mongo"
"github.com/umputun/remark/backend/app/store/service"
)
backend/app/store/avatar/gridfs.go
+1 -1
View File
@@ -7,10 +7,10 @@ import (
"log"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine/mongo"
)
// NewGridFS makes gridfs (mongo) avatar store
backend/app/store/avatar/gridfs_test.go
+1 -2
View File
@@ -6,10 +6,9 @@ import (
"testing"
"github.com/globalsign/mgo"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store/engine/mongo"
)
func TestGridFS_PutAndGet(t *testing.T) {
backend/app/store/engine/mongo.go
+1 -1
View File
@@ -5,11 +5,11 @@ import (
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/go-pkgz/mongo"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine/mongo"
)
// Mongo implements engine interface
backend/app/store/engine/mongo/connection_test.go
-126
View File
@@ -1,126 +0,0 @@
package mongo
import (
"fmt"
"testing"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/hashicorp/go-multierror"
"github.com/stretchr/testify/assert"
)
type testRecord struct {
Symbol string
Num int
}
func TestConnection_WithCollection(t *testing.T) {
c, err := write(t)
if err != nil {
return
}
defer RemoveTestCollection(t, c)
var res []testRecord
err = c.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 100, len(res))
err = c.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"symbol": "blah"}).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
r1 := testRecord{}
err = c.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"symbol": "blah"}).One(&r1)
})
assert.Equal(t, mgo.ErrNotFound, err)
c = NewConnection(c.server, "test", "bbbbbbbaaad")
err = c.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(bson.M{"symbol": "blah"}).One(&r1)
})
assert.Equal(t, mgo.ErrNotFound, err)
}
func TestConnection_WithCollectionNoDB(t *testing.T) {
c, err := write(t)
if err != nil {
return
}
defer RemoveTestCollection(t, c)
var res []testRecord
err = c.WithCollection(func(coll *mgo.Collection) error {
return coll.Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 100, len(res))
}
func TestConnection_WithDB(t *testing.T) {
c, err := write(t)
if err != nil {
return
}
defer RemoveTestCollection(t, c)
var res []testRecord
err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
return dbase.C(c.collection).Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 100, len(res))
err = c.WithDB(func(dbase *mgo.Database) error {
return dbase.C(c.collection).Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 100, len(res))
}
func TestCleanup(t *testing.T) {
c, err := write(t)
if err != nil {
return
}
var res []testRecord
err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
return dbase.C(c.collection).Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 100, len(res))
RemoveTestCollections(t, c, c.collection)
err = c.WithCustomDB("test", func(dbase *mgo.Database) error {
return dbase.C(c.collection).Find(nil).All(&res)
})
assert.Nil(t, err)
assert.Equal(t, 0, len(res))
}
func write(t *testing.T) (*Connection, error) {
c, err := MakeTestConnection(t)
if err != nil {
return nil, err
}
err = c.WithCollection(func(coll *mgo.Collection) error {
errs := new(multierror.Error)
for i := 0; i < 100; i++ {
r := testRecord{
Symbol: fmt.Sprintf("symb-%02d", i%5),
Num: i,
}
insertErr := coll.Insert(r)
assert.Nil(t, insertErr, fmt.Sprintf("insert %+v", r))
errs = multierror.Append(errs, insertErr)
}
return errs.ErrorOrNil()
})
return c, err
}
backend/app/store/engine/mongo/server_test.go
-93
View File
@@ -1,93 +0,0 @@
package mongo
import (
"os"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/stretchr/testify/assert"
)
func TestServer_NewServerGood(t *testing.T) {
mongoURL := os.Getenv("MONGO_REMARK_TEST")
if mongoURL == "" || mongoURL == "skip" {
t.Skip("no MONGO_REMARK_TEST in env")
return
}
m, err := NewServerWithURL(mongoURL, 3*time.Second)
assert.Nil(t, err)
assert.NotNil(t, m)
assert.True(t, m.String() != "")
}
func TestServer_NewServerBad(t *testing.T) {
_, err := NewServerWithURL("mongodb://127.0.0.3:27017/test", 100*time.Millisecond)
assert.NotNil(t, err)
t.Log(err)
_, err = NewServer(mgo.DialInfo{Addrs: []string{"127.0.0.2"}, Timeout: 100 * time.Millisecond}, ServerParams{})
assert.NotNil(t, err)
_, err = NewServer(mgo.DialInfo{}, ServerParams{})
assert.NotNil(t, err)
_, err = NewServerWithURL("mongodb://mongo:27017/test?blah=xxx", 100*time.Millisecond)
assert.NotNil(t, err)
t.Log(err)
}
func TestServer_parse(t *testing.T) {
tbl := []struct {
mongoURL string
timeout time.Duration
params ServerParams
dial mgo.DialInfo
isErr bool
}{
{
"mongodb://127.0.0.3:27017/test", time.Millisecond,
ServerParams{ConsistencyMode: 1},
mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test",
ReadPreference: &mgo.ReadPreference{Mode: 2}},
false,
},
{
"mongodb://user:passwd@127.0.0.3:27017/test?ssl=true&authSource=admin", time.Millisecond,
ServerParams{ConsistencyMode: 1, SSL: true},
mgo.DialInfo{Addrs: []string{"127.0.0.3:27017"}, Timeout: 1000000, Database: "test", Source: "admin",
Username: "user", Password: "passwd", ReadPreference: &mgo.ReadPreference{Mode: 2}},
false,
},
{
"mongodb://127.0.0.3", time.Millisecond,
ServerParams{ConsistencyMode: 1, SSL: false},
mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}},
false,
},
{
"127.0.0.3", time.Millisecond,
ServerParams{ConsistencyMode: 1, SSL: false},
mgo.DialInfo{Addrs: []string{"127.0.0.3"}, Timeout: 1000000, ReadPreference: &mgo.ReadPreference{Mode: 2}},
false,
},
{
"127.0.0.3?xxx=yyy", time.Millisecond,
ServerParams{}, mgo.DialInfo{},
true,
},
}
for i, tt := range tbl {
dial, params, err := parseURL(tt.mongoURL, tt.timeout)
dial.DialServer = nil
if tt.isErr {
assert.NotNil(t, err, "expect error #%d", i)
t.Logf("dial %+v, params %+v", dial, params)
continue
}
assert.Equal(t, tt.dial, dial, "test #%d", i)
assert.Equal(t, tt.params, params, "test #%d", i)
}
}
backend/app/store/engine/mongo/writer_test.go
-162
View File
@@ -1,162 +0,0 @@
package mongo
import (
"sync"
"testing"
"time"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestWriter(t *testing.T) {
count := func(conn *Connection) (res int) {
_ = conn.WithCollection(func(coll *mgo.Collection) error {
var err error
res, err = coll.Find(nil).Count()
assert.Nil(t, err)
return nil
})
return res
}
conn, err := MakeTestConnection(t)
if err != nil {
return
}
defer RemoveTestCollection(t, conn)
var wr BufferedWriter = NewBufferedWriter(3, conn)
assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1")
assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2")
assert.Equal(t, 0, count(conn), "nothing yet")
assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3")
assert.Equal(t, 3, count(conn), "all 3 records in")
assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4")
assert.Equal(t, 3, count(conn), "still 3 records")
assert.Nil(t, wr.Flush())
assert.Equal(t, 4, count(conn), "all 4 records")
assert.Nil(t, wr.Flush())
assert.Equal(t, 4, count(conn), "still 4 records, nothing left to flush")
assert.Nil(t, wr.Close())
}
func TestWriter_Parallel(t *testing.T) {
conn, err := MakeTestConnection(t)
if err != nil {
return
}
defer RemoveTestCollection(t, conn)
var wg sync.WaitGroup
wr := NewBufferedWriter(75, conn)
writeMany := func() {
for i := 0; i < 100; i++ {
require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2}))
}
wr.Flush()
wg.Done()
}
for i := 0; i < 16; i++ {
wg.Add(1)
go writeMany()
}
wg.Wait()
_ = conn.WithCollection(func(coll *mgo.Collection) error {
res, err := coll.Find(nil).Count()
assert.Nil(t, err)
assert.Equal(t, 100*16, res)
return nil
})
assert.Nil(t, wr.Close())
}
func TestWriter_WithAuthFlush(t *testing.T) {
conn, err := MakeTestConnection(t)
if err != nil {
return
}
defer RemoveTestCollection(t, conn)
var wr BufferedWriter = NewBufferedWriter(3, conn).WithAutoFlush(500 * time.Millisecond)
count := func() (res int) {
_ = conn.WithCollection(func(coll *mgo.Collection) error {
var err error
res, err = coll.Find(nil).Count()
assert.Nil(t, err)
return nil
})
return res
}
assert.Nil(t, wr.Write(bson.M{"key1": "val1"}), "write rec #1")
assert.Nil(t, wr.Write(bson.M{"key2": "val2"}), "write rec #2")
assert.Equal(t, 0, count(), "nothing yet")
time.Sleep(600 * time.Millisecond)
assert.Equal(t, 2, count(), "2 records flushed")
assert.Nil(t, wr.Write(bson.M{"key3": "val3"}), "write rec #3")
assert.Nil(t, wr.Write(bson.M{"key4": "val4"}), "write rec #4")
assert.Nil(t, wr.Write(bson.M{"key5": "val5"}), "write rec #5")
assert.Equal(t, 5, count(), "5 records, flushed by size, not duration")
assert.Nil(t, wr.Write(bson.M{"key6": "val6"}), "write rec #6")
assert.Nil(t, wr.Write(bson.M{"key7": "val7"}), "write rec #7")
assert.Equal(t, 5, count(), "still 5 records")
assert.Nil(t, wr.Flush())
assert.Equal(t, 7, count(), "all 7 records")
assert.Nil(t, wr.Flush())
assert.Equal(t, 7, count(), "still 7 records, nothing left to flush")
assert.Nil(t, wr.Close())
}
func TestWriter_ParallelWithAutoFlush(t *testing.T) {
conn, err := MakeTestConnection(t)
if err != nil {
return
}
defer RemoveTestCollection(t, conn)
var wg sync.WaitGroup
wr := NewBufferedWriter(75, conn).WithAutoFlush(time.Millisecond)
writeMany := func() {
for i := 0; i < 100; i++ {
require.NoError(t, wr.Write(bson.M{"key1": 1, "key2": 2}))
time.Sleep(time.Millisecond * 3)
}
wr.Flush()
wg.Done()
}
for i := 0; i < 16; i++ {
wg.Add(1)
go writeMany()
}
wg.Wait()
_ = conn.WithCollection(func(coll *mgo.Collection) error {
res, err := coll.Find(nil).Count()
assert.Nil(t, err)
assert.Equal(t, 100*16, res)
return nil
})
assert.Nil(t, wr.Close())
}
backend/app/store/engine/mongo_test.go
+1 -1
View File
@@ -6,11 +6,11 @@ import (
"testing"
"time"
"github.com/go-pkgz/mongo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/umputun/remark/backend/app/store"
"github.com/umputun/remark/backend/app/store/engine/mongo"
)
func TestMongo_CreateAndFind(t *testing.T) {
backend/vendor/github.com/go-pkgz/mongo/.gitignore
Generated
+12
View File
@@ -0,0 +1,12 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
backend/vendor/github.com/go-pkgz/mongo/.travis.yml
Generated Vendored
+23
View File
@@ -0,0 +1,23 @@
language: go
go:
- "1.10.x"
go_import_path: github.com/go-pkgz/mongo
services: mongodb
before_install:
- cp -rf .vendor vendor
- go get github.com/mattn/goveralls
- go get gopkg.in/alecthomas/gometalinter.v2
- $GOPATH/bin/gometalinter.v2 --install
before_script:
- sleep 3
script:
- export MONGO_TEST=mongodb://127.0.0.1:27017
- go test ./...
- $GOPATH/bin/gometalinter.v2 --exclude=test --exclude=mock --exclude=vendor ./...
- $GOPATH/bin/goveralls -service=travis-ci
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/davecgh/go-spew/LICENSE
Generated Vendored
+15
View File
@@ -0,0 +1,15 @@
ISC License
Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/LICENSE
Generated Vendored
+25
View File
@@ -0,0 +1,25 @@
mgo - MongoDB driver for Go
Copyright (c) 2010-2013 - Gustavo Niemeyer <gustavo@niemeyer.net>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/bson/LICENSE
Generated Vendored
+25
View File
@@ -0,0 +1,25 @@
BSON library for Go
Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/globalsign/mgo/internal/json/LICENSE
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/pmezard/go-difflib/LICENSE
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2013, Patrick Mezard
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
The names of its contributors may not be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
backend/vendor/github.com/go-pkgz/mongo/.vendor/github.com/stretchr/testify/LICENSE
Generated Vendored
+22
View File
@@ -0,0 +1,22 @@
Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell
Please consider promoting this project if you find it useful.
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of the Software,
and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
backend/vendor/github.com/go-pkgz/mongo/LICENSE
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018 Umputun
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
backend/vendor/github.com/go-pkgz/mongo/README.md
Generated Vendored
+58
View File
@@ -0,0 +1,58 @@
# Mongo [![Build Status](https://travis-ci.org/go-pkgz/mongo.svg?branch=master)](https://travis-ci.org/go-pkgz/mongo) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/mongo)](https://goreportcard.com/report/github.com/go-pkgz/mongo) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/mongo/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/mongo?branch=master)
Provides helpers on top of [mgo](https://github.com/globalsign/mgo)
## Install and update
`go get -u github.com/go-pkgz/mongo`
## Usage
- `Server` represents mongo instance and provides session accessor. Application usually creates one server object and uses it for anything needed with this particular mongo host or replica set.
- `Connection` encapsulates session and provides auto-closable wrapper. Each requests runs inside one of With* function makes new mongo session and closes on completion.
- `BufferedWriter` implements buffered writer to mongo. Write method caching internally till it reached buffer size. Flush methods can be called manually at any time.
```golang
m, err := NewServerWithURL("mongodb://127.0.0.1:27017/test?debug=true", 3*time.Second)
if err != nil {
panic("can't make mongo server")
}
type testRecord struct {
Key1 string
Kay2 int
}
err = c.WithCollection(func(coll *mgo.Collection) error { // create session
// insert 100 records
for i := 0; i < 100; i++ {
r := testRecord{
Key1: fmt.Sprintf("key-%02d", i%5),
Key2: i,
}
if e := coll.Insert(r); e != nil {
return e
}
}
return nil
})
```
## Dependencies
- [globalsign/mgo](https://github.com/globalsign/mgo) - mgo mongo driver
- [stretchr/testify/](https://github.com/stretchr/testify) - testing library (test-only dependency)
## Testing
`testing.go` helps to create test for real mongo (not mocks)
- `mongo.MakeTestConnection` creates `mongo.Connection` for url defined in env `MONGO_TEST`. If not defined `mongodb://mongo:27017` used. By default it will use random connection with prefix `test_` in `test` DB.
- `mongo.RemoveTestCollection` - drops collection used by `MakeTestConnection`
- `mongo.RemoveTestCollections` - drops user-defined collections from `test` DB
backend/app/store/engine/mongo/connection.go → backend/vendor/github.com/go-pkgz/mongo/connection.go
Generated Vendored
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/globalsign/mgo"
)
// sessionFn is a function for all With calls, terminator not supported
// sessionFn is a function for all With*Collection calls
type sessionFn func(coll *mgo.Collection) error
// Connection allows to run request in separate session, closing automatically
backend/app/store/engine/mongo/server.go → backend/vendor/github.com/go-pkgz/mongo/server.go
Generated Vendored
+7 -11
View File
@@ -1,9 +1,10 @@
// Package mongo wraps mgo to provide easier way to construct mongo server (with auth).
// Connection provides With* func warapers to run query with session copy
// Connection provides With* func wrappers to run query with session copy
package mongo
import (
"crypto/tls"
"errors"
"fmt"
"log"
"net"
@@ -12,7 +13,6 @@ import (
"time"
"github.com/globalsign/mgo"
"github.com/pkg/errors"
)
// Server represents mongo instance and provides session accessor
@@ -24,9 +24,9 @@ type Server struct {
// ServerParams optional set of parameters
type ServerParams struct {
ConsistencyMode mgo.Mode
Delay int // initial delay to give mongo server some time to start, in case if mongo part of the same compose
Debug bool // turn on mgo debug mode
ConsistencyMode mgo.Mode
SSL bool
}
@@ -35,14 +35,13 @@ type ServerParams struct {
func NewServerWithURL(url string, timeout time.Duration) (res *Server, err error) {
dial, params, err := parseURL(url, timeout)
if err != nil {
return nil, errors.Wrapf(err, "failed to create mongo server from url %s", url)
return nil, fmt.Errorf("failed to create mongo server from url %s, %s", url, err)
}
return NewServer(dial, params)
}
// NewServer doing auth if passwd != "" and can delay to make sure local mongo is up
func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) {
log.Printf("[INFO] make new mongo server %v with %+v", dial.Addrs, params)
result := Server{dial: dial, params: params}
if params.Debug {
@@ -71,9 +70,7 @@ func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error)
session, err := mgo.DialWithInfo(&dial)
if err != nil {
err = fmt.Errorf("can't connect to mongo, %v", err)
log.Printf("[ERROR] %v", err)
return nil, err
return nil, fmt.Errorf("can't connect to mongo, %v", err)
}
session.SetMode(params.ConsistencyMode, true)
session.SetSyncTimeout(30 * time.Second)
@@ -83,8 +80,7 @@ func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error)
creds := &mgo.Credential{Username: dial.Username, Password: dial.Password, Source: dial.Source}
log.Printf("[DEBUG] login to mongo, user=%s, db=%s", creds.Username, creds.Source)
if err = session.Login(creds); err != nil {
log.Printf("[ERROR] can't login to mongo, %v", err)
return nil, err
return nil, fmt.Errorf("can't login to mongo, %v", err)
}
}
@@ -114,7 +110,7 @@ func parseURL(mongoURL string, connectTimeout time.Duration) (mgo.DialInfo, Serv
dial, err := mgo.ParseURL(mongoURL)
if err != nil {
return mgo.DialInfo{}, ServerParams{}, errors.Wrapf(err, "failed to pars mongo url %s", mongoURL)
return mgo.DialInfo{}, ServerParams{}, fmt.Errorf("failed to pars mongo url %s, %s", mongoURL, err)
}
dial.Timeout = connectTimeout
return *dial, params, nil
backend/app/store/engine/mongo/testing.go → backend/vendor/github.com/go-pkgz/mongo/testing.go
Generated Vendored
+16 -13
View File
@@ -9,7 +9,7 @@ import (
"time"
"github.com/globalsign/mgo"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -17,24 +17,15 @@ import (
var conn *Connection
var once sync.Once
// MakeTestConnection connects to MONGO_REMARK_TEST url or "mongo" host (in no env) and returns new connection.
// MakeTestConnection connects to MONGO_TEST url or "mongo" host (in no env) and returns new connection.
// collection name randomized on each call
func MakeTestConnection(t *testing.T) (*Connection, error) {
mongoURL := os.Getenv("MONGO_REMARK_TEST")
if mongoURL == "" {
t.Skip("no MONGO_REMARK_TEST in env")
return nil, errors.New("skip")
}
if mongoURL == "skip" {
t.Skip("skip mongo test")
return nil, errors.New("skip")
}
mongoURL := getMongoURL(t)
once.Do(func() {
log.Print("[DEBUG] connect to mongo test instance")
srv, err := NewServerWithURL(mongoURL, 10*time.Second)
assert.Nil(t, err, "failed to dial")
collName := fmt.Sprintf("remark42_test_%d", time.Now().Nanosecond())
collName := fmt.Sprintf("test_%d", time.Now().Nanosecond())
conn = NewConnection(srv, "test", collName)
})
RemoveTestCollection(t, conn)
@@ -71,3 +62,15 @@ func RemoveTestCollections(t *testing.T, c *Connection, collections ...string) {
}
}
func getMongoURL(t *testing.T) string {
mongoURL := os.Getenv("MONGO_TEST")
if mongoURL == "" {
mongoURL = "mongodb://mongo:27017"
t.Logf("no MONGO_TEST in env, defaulted to %s", mongoURL)
}
if mongoURL == "skip" {
t.Skip("skip mongo test")
}
return mongoURL
}
backend/app/store/engine/mongo/writer.go → backend/vendor/github.com/go-pkgz/mongo/writer.go
Generated Vendored
+10 -7
View File
@@ -2,12 +2,12 @@ package mongo
import (
"context"
"fmt"
"log"
"sync"
"time"
"github.com/globalsign/mgo"
"github.com/pkg/errors"
)
// BufferedWriter defines interface for writes and flush
@@ -76,7 +76,6 @@ func (bw *BufferedWriterMgo) WithAutoFlush(duration time.Duration) *BufferedWrit
}
}
case <-bw.ctx.Done():
log.Printf("[DEBUG] mongo writer flusher terminated")
return
}
}
@@ -92,9 +91,10 @@ func (bw *BufferedWriterMgo) Write(rec interface{}) error {
bw.lastWriteTime = time.Now()
bw.buffer = append(bw.buffer, rec)
if len(bw.buffer) >= bw.bufferSize {
err := bw.writeBuffer()
if err := bw.writeBuffer(); err != nil {
return fmt.Errorf("failed to write to %s, %s", bw.connection, err)
}
bw.buffer = bw.buffer[0:0]
return errors.Wrapf(err, "failed to write to %s", bw.connection)
}
return nil
})
@@ -102,11 +102,15 @@ func (bw *BufferedWriterMgo) Write(rec interface{}) error {
// Flush writes everything left in buffer to mongo
func (bw *BufferedWriterMgo) Flush() error {
return bw.synced(func() error {
err := bw.synced(func() error {
err := bw.writeBuffer()
bw.buffer = bw.buffer[0:0]
return errors.Wrapf(err, "failed to flush to %s", bw.connection)
return err
})
if err != nil {
return fmt.Errorf("failed to flush to %s, %s", bw.connection, err)
}
return nil
}
// Close flushes all in-fly records and terminates background auto-flusher
@@ -116,7 +120,6 @@ func (bw *BufferedWriterMgo) Close() (err error) {
if bw.flushDuration > 0 {
bw.cancel()
<-bw.ctx.Done()
log.Printf("[DEBUG] mongo buffered writer closed")
}
return err
})
compose-dev-backend.yml
+1 -1
View File
@@ -18,7 +18,7 @@ services:
dockerfile: Dockerfile
args:
- SKIP_FRONTEND_TEST=true
- MONGO_REMARK_TEST=skip # disable mongo tests on build by default. To allow remove =skip part and see above
- MONGO_TEST=skip # disable mongo tests on build by default. To allow remove =skip part and see above
image: umputun/remark42:dev
container_name: "remark42-dev"