update deps
This commit is contained in:
Umputun
+2
@@ -5,3 +5,5 @@
|
||||
The Cascadia package implements CSS selectors for use with the parse trees produced by the html package.
|
||||
|
||||
To test CSS selectors without writing Go code, check out [cascadia](https://github.com/suntong/cascadia) the command line tool, a thin wrapper around this package.
|
||||
|
||||
[Refer to godoc here](https://godoc.org/github.com/andybalholm/cascadia).
|
||||
|
||||
+123
-157
@@ -7,8 +7,6 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// a parser for CSS selectors
|
||||
@@ -56,6 +54,26 @@ func (p *parser) parseEscape() (result string, err error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// toLowerASCII returns s with all ASCII capital letters lowercased.
|
||||
func toLowerASCII(s string) string {
|
||||
var b []byte
|
||||
for i := 0; i < len(s); i++ {
|
||||
if c := s[i]; 'A' <= c && c <= 'Z' {
|
||||
if b == nil {
|
||||
b = make([]byte, len(s))
|
||||
copy(b, s)
|
||||
}
|
||||
b[i] = s[i] + ('a' - 'A')
|
||||
}
|
||||
}
|
||||
|
||||
if b == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func hexDigit(c byte) bool {
|
||||
return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F'
|
||||
}
|
||||
@@ -280,92 +298,92 @@ func (p *parser) consumeClosingParenthesis() bool {
|
||||
}
|
||||
|
||||
// parseTypeSelector parses a type selector (one that matches by tag name).
|
||||
func (p *parser) parseTypeSelector() (result Selector, err error) {
|
||||
func (p *parser) parseTypeSelector() (result tagSelector, err error) {
|
||||
tag, err := p.parseIdentifier()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
|
||||
return typeSelector(tag), nil
|
||||
return tagSelector{tag: toLowerASCII(tag)}, nil
|
||||
}
|
||||
|
||||
// parseIDSelector parses a selector that matches by id attribute.
|
||||
func (p *parser) parseIDSelector() (Selector, error) {
|
||||
func (p *parser) parseIDSelector() (idSelector, error) {
|
||||
if p.i >= len(p.s) {
|
||||
return nil, fmt.Errorf("expected id selector (#id), found EOF instead")
|
||||
return idSelector{}, fmt.Errorf("expected id selector (#id), found EOF instead")
|
||||
}
|
||||
if p.s[p.i] != '#' {
|
||||
return nil, fmt.Errorf("expected id selector (#id), found '%c' instead", p.s[p.i])
|
||||
return idSelector{}, fmt.Errorf("expected id selector (#id), found '%c' instead", p.s[p.i])
|
||||
}
|
||||
|
||||
p.i++
|
||||
id, err := p.parseName()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return idSelector{}, err
|
||||
}
|
||||
|
||||
return attributeEqualsSelector("id", id), nil
|
||||
return idSelector{id: id}, nil
|
||||
}
|
||||
|
||||
// parseClassSelector parses a selector that matches by class attribute.
|
||||
func (p *parser) parseClassSelector() (Selector, error) {
|
||||
func (p *parser) parseClassSelector() (classSelector, error) {
|
||||
if p.i >= len(p.s) {
|
||||
return nil, fmt.Errorf("expected class selector (.class), found EOF instead")
|
||||
return classSelector{}, fmt.Errorf("expected class selector (.class), found EOF instead")
|
||||
}
|
||||
if p.s[p.i] != '.' {
|
||||
return nil, fmt.Errorf("expected class selector (.class), found '%c' instead", p.s[p.i])
|
||||
return classSelector{}, fmt.Errorf("expected class selector (.class), found '%c' instead", p.s[p.i])
|
||||
}
|
||||
|
||||
p.i++
|
||||
class, err := p.parseIdentifier()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return classSelector{}, err
|
||||
}
|
||||
|
||||
return attributeIncludesSelector("class", class), nil
|
||||
return classSelector{class: class}, nil
|
||||
}
|
||||
|
||||
// parseAttributeSelector parses a selector that matches by attribute value.
|
||||
func (p *parser) parseAttributeSelector() (Selector, error) {
|
||||
func (p *parser) parseAttributeSelector() (attrSelector, error) {
|
||||
if p.i >= len(p.s) {
|
||||
return nil, fmt.Errorf("expected attribute selector ([attribute]), found EOF instead")
|
||||
return attrSelector{}, fmt.Errorf("expected attribute selector ([attribute]), found EOF instead")
|
||||
}
|
||||
if p.s[p.i] != '[' {
|
||||
return nil, fmt.Errorf("expected attribute selector ([attribute]), found '%c' instead", p.s[p.i])
|
||||
return attrSelector{}, fmt.Errorf("expected attribute selector ([attribute]), found '%c' instead", p.s[p.i])
|
||||
}
|
||||
|
||||
p.i++
|
||||
p.skipWhitespace()
|
||||
key, err := p.parseIdentifier()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return attrSelector{}, err
|
||||
}
|
||||
key = toLowerASCII(key)
|
||||
|
||||
p.skipWhitespace()
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in attribute selector")
|
||||
return attrSelector{}, errors.New("unexpected EOF in attribute selector")
|
||||
}
|
||||
|
||||
if p.s[p.i] == ']' {
|
||||
p.i++
|
||||
return attributeExistsSelector(key), nil
|
||||
return attrSelector{key: key, operation: ""}, nil
|
||||
}
|
||||
|
||||
if p.i+2 >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in attribute selector")
|
||||
return attrSelector{}, errors.New("unexpected EOF in attribute selector")
|
||||
}
|
||||
|
||||
op := p.s[p.i : p.i+2]
|
||||
if op[0] == '=' {
|
||||
op = "="
|
||||
} else if op[1] != '=' {
|
||||
return nil, fmt.Errorf(`expected equality operator, found "%s" instead`, op)
|
||||
return attrSelector{}, fmt.Errorf(`expected equality operator, found "%s" instead`, op)
|
||||
}
|
||||
p.i += len(op)
|
||||
|
||||
p.skipWhitespace()
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in attribute selector")
|
||||
return attrSelector{}, errors.New("unexpected EOF in attribute selector")
|
||||
}
|
||||
var val string
|
||||
var rx *regexp.Regexp
|
||||
@@ -380,46 +398,32 @@ func (p *parser) parseAttributeSelector() (Selector, error) {
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return attrSelector{}, err
|
||||
}
|
||||
|
||||
p.skipWhitespace()
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in attribute selector")
|
||||
return attrSelector{}, errors.New("unexpected EOF in attribute selector")
|
||||
}
|
||||
if p.s[p.i] != ']' {
|
||||
return nil, fmt.Errorf("expected ']', found '%c' instead", p.s[p.i])
|
||||
return attrSelector{}, fmt.Errorf("expected ']', found '%c' instead", p.s[p.i])
|
||||
}
|
||||
p.i++
|
||||
|
||||
switch op {
|
||||
case "=":
|
||||
return attributeEqualsSelector(key, val), nil
|
||||
case "!=":
|
||||
return attributeNotEqualSelector(key, val), nil
|
||||
case "~=":
|
||||
return attributeIncludesSelector(key, val), nil
|
||||
case "|=":
|
||||
return attributeDashmatchSelector(key, val), nil
|
||||
case "^=":
|
||||
return attributePrefixSelector(key, val), nil
|
||||
case "$=":
|
||||
return attributeSuffixSelector(key, val), nil
|
||||
case "*=":
|
||||
return attributeSubstringSelector(key, val), nil
|
||||
case "#=":
|
||||
return attributeRegexSelector(key, rx), nil
|
||||
case "=", "!=", "~=", "|=", "^=", "$=", "*=", "#=":
|
||||
return attrSelector{key: key, val: val, operation: op, regexp: rx}, nil
|
||||
default:
|
||||
return attrSelector{}, fmt.Errorf("attribute operator %q is not supported", op)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("attribute operator %q is not supported", op)
|
||||
}
|
||||
|
||||
var errExpectedParenthesis = errors.New("expected '(' but didn't find it")
|
||||
var errExpectedClosingParenthesis = errors.New("expected ')' but didn't find it")
|
||||
var errUnmatchedParenthesis = errors.New("unmatched '('")
|
||||
|
||||
// parsePseudoclassSelector parses a pseudoclass selector like :not(p).
|
||||
func (p *parser) parsePseudoclassSelector() (Selector, error) {
|
||||
// parsePseudoclassSelector parses a pseudoclass selector like :not(p)
|
||||
func (p *parser) parsePseudoclassSelector() (out Sel, err error) {
|
||||
if p.i >= len(p.s) {
|
||||
return nil, fmt.Errorf("expected pseudoclass selector (:pseudoclass), found EOF instead")
|
||||
}
|
||||
@@ -428,40 +432,36 @@ func (p *parser) parsePseudoclassSelector() (Selector, error) {
|
||||
}
|
||||
|
||||
p.i++
|
||||
if p.s[p.i] == ':' { // we found a pseudo-element
|
||||
p.i++
|
||||
}
|
||||
|
||||
name, err := p.parseIdentifier()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
name = toLowerASCII(name)
|
||||
|
||||
switch name {
|
||||
case "not", "has", "haschild":
|
||||
if !p.consumeParenthesis() {
|
||||
return nil, errExpectedParenthesis
|
||||
return out, errExpectedParenthesis
|
||||
}
|
||||
sel, parseErr := p.parseSelectorGroup()
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
return out, parseErr
|
||||
}
|
||||
if !p.consumeClosingParenthesis() {
|
||||
return nil, errExpectedClosingParenthesis
|
||||
return out, errExpectedClosingParenthesis
|
||||
}
|
||||
|
||||
switch name {
|
||||
case "not":
|
||||
return negatedSelector(sel), nil
|
||||
case "has":
|
||||
return hasDescendantSelector(sel), nil
|
||||
case "haschild":
|
||||
return hasChildSelector(sel), nil
|
||||
}
|
||||
out = relativePseudoClassSelector{name: name, match: sel}
|
||||
|
||||
case "contains", "containsown":
|
||||
if !p.consumeParenthesis() {
|
||||
return nil, errExpectedParenthesis
|
||||
return out, errExpectedParenthesis
|
||||
}
|
||||
if p.i == len(p.s) {
|
||||
return nil, errUnmatchedParenthesis
|
||||
return out, errUnmatchedParenthesis
|
||||
}
|
||||
var val string
|
||||
switch p.s[p.i] {
|
||||
@@ -471,95 +471,75 @@ func (p *parser) parsePseudoclassSelector() (Selector, error) {
|
||||
val, err = p.parseIdentifier()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return out, err
|
||||
}
|
||||
val = strings.ToLower(val)
|
||||
p.skipWhitespace()
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in pseudo selector")
|
||||
return out, errors.New("unexpected EOF in pseudo selector")
|
||||
}
|
||||
if !p.consumeClosingParenthesis() {
|
||||
return nil, errExpectedClosingParenthesis
|
||||
return out, errExpectedClosingParenthesis
|
||||
}
|
||||
|
||||
switch name {
|
||||
case "contains":
|
||||
return textSubstrSelector(val), nil
|
||||
case "containsown":
|
||||
return ownTextSubstrSelector(val), nil
|
||||
}
|
||||
out = containsPseudoClassSelector{own: name == "containsown", value: val}
|
||||
|
||||
case "matches", "matchesown":
|
||||
if !p.consumeParenthesis() {
|
||||
return nil, errExpectedParenthesis
|
||||
return out, errExpectedParenthesis
|
||||
}
|
||||
rx, err := p.parseRegex()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return out, err
|
||||
}
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("unexpected EOF in pseudo selector")
|
||||
return out, errors.New("unexpected EOF in pseudo selector")
|
||||
}
|
||||
if !p.consumeClosingParenthesis() {
|
||||
return nil, errExpectedClosingParenthesis
|
||||
return out, errExpectedClosingParenthesis
|
||||
}
|
||||
|
||||
switch name {
|
||||
case "matches":
|
||||
return textRegexSelector(rx), nil
|
||||
case "matchesown":
|
||||
return ownTextRegexSelector(rx), nil
|
||||
}
|
||||
out = regexpPseudoClassSelector{own: name == "matchesown", regexp: rx}
|
||||
|
||||
case "nth-child", "nth-last-child", "nth-of-type", "nth-last-of-type":
|
||||
if !p.consumeParenthesis() {
|
||||
return nil, errExpectedParenthesis
|
||||
return out, errExpectedParenthesis
|
||||
}
|
||||
a, b, err := p.parseNth()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return out, err
|
||||
}
|
||||
if !p.consumeClosingParenthesis() {
|
||||
return nil, errExpectedClosingParenthesis
|
||||
return out, errExpectedClosingParenthesis
|
||||
}
|
||||
if a == 0 {
|
||||
switch name {
|
||||
case "nth-child":
|
||||
return simpleNthChildSelector(b, false), nil
|
||||
case "nth-of-type":
|
||||
return simpleNthChildSelector(b, true), nil
|
||||
case "nth-last-child":
|
||||
return simpleNthLastChildSelector(b, false), nil
|
||||
case "nth-last-of-type":
|
||||
return simpleNthLastChildSelector(b, true), nil
|
||||
}
|
||||
}
|
||||
return nthChildSelector(a, b,
|
||||
name == "nth-last-child" || name == "nth-last-of-type",
|
||||
name == "nth-of-type" || name == "nth-last-of-type"),
|
||||
nil
|
||||
last := name == "nth-last-child" || name == "nth-last-of-type"
|
||||
ofType := name == "nth-of-type" || name == "nth-last-of-type"
|
||||
out = nthPseudoClassSelector{a: a, b: b, last: last, ofType: ofType}
|
||||
|
||||
case "first-child":
|
||||
return simpleNthChildSelector(1, false), nil
|
||||
out = nthPseudoClassSelector{a: 0, b: 1, ofType: false, last: false}
|
||||
case "last-child":
|
||||
return simpleNthLastChildSelector(1, false), nil
|
||||
out = nthPseudoClassSelector{a: 0, b: 1, ofType: false, last: true}
|
||||
case "first-of-type":
|
||||
return simpleNthChildSelector(1, true), nil
|
||||
out = nthPseudoClassSelector{a: 0, b: 1, ofType: true, last: false}
|
||||
case "last-of-type":
|
||||
return simpleNthLastChildSelector(1, true), nil
|
||||
out = nthPseudoClassSelector{a: 0, b: 1, ofType: true, last: true}
|
||||
case "only-child":
|
||||
return onlyChildSelector(false), nil
|
||||
out = onlyChildPseudoClassSelector{ofType: false}
|
||||
case "only-of-type":
|
||||
return onlyChildSelector(true), nil
|
||||
out = onlyChildPseudoClassSelector{ofType: true}
|
||||
case "input":
|
||||
return inputSelector, nil
|
||||
out = inputPseudoClassSelector{}
|
||||
case "empty":
|
||||
return emptyElementSelector, nil
|
||||
out = emptyElementPseudoClassSelector{}
|
||||
case "root":
|
||||
return rootSelector, nil
|
||||
out = rootPseudoClassSelector{}
|
||||
case "after", "backdrop", "before", "cue", "first-letter", "first-line", "grammar-error", "marker", "placeholder", "selection", "spelling-error":
|
||||
return out, errors.New("pseudo-elements are not yet supported")
|
||||
default:
|
||||
return out, fmt.Errorf("unknown pseudoclass or pseudoelement :%s", name)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unknown pseudoclass :%s", name)
|
||||
return
|
||||
}
|
||||
|
||||
// parseInteger parses a decimal integer.
|
||||
@@ -705,8 +685,8 @@ invalid:
|
||||
|
||||
// parseSimpleSelectorSequence parses a selector sequence that applies to
|
||||
// a single element.
|
||||
func (p *parser) parseSimpleSelectorSequence() (Selector, error) {
|
||||
var result Selector
|
||||
func (p *parser) parseSimpleSelectorSequence() (Sel, error) {
|
||||
var selectors []Sel
|
||||
|
||||
if p.i >= len(p.s) {
|
||||
return nil, errors.New("expected selector, found EOF instead")
|
||||
@@ -723,13 +703,15 @@ func (p *parser) parseSimpleSelectorSequence() (Selector, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = r
|
||||
selectors = append(selectors, r)
|
||||
}
|
||||
|
||||
loop:
|
||||
for p.i < len(p.s) {
|
||||
var ns Selector
|
||||
var err error
|
||||
var (
|
||||
ns Sel
|
||||
err error
|
||||
)
|
||||
switch p.s[p.i] {
|
||||
case '#':
|
||||
ns, err = p.parseIDSelector()
|
||||
@@ -745,37 +727,33 @@ loop:
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if result == nil {
|
||||
result = ns
|
||||
} else {
|
||||
result = intersectionSelector(result, ns)
|
||||
}
|
||||
}
|
||||
|
||||
if result == nil {
|
||||
result = func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode
|
||||
}
|
||||
selectors = append(selectors, ns)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
if len(selectors) == 1 { // no need wrap the selectors in compoundSelector
|
||||
return selectors[0], nil
|
||||
}
|
||||
return compoundSelector{selectors: selectors}, nil
|
||||
}
|
||||
|
||||
// parseSelector parses a selector that may include combinators.
|
||||
func (p *parser) parseSelector() (result Selector, err error) {
|
||||
func (p *parser) parseSelector() (Sel, error) {
|
||||
p.skipWhitespace()
|
||||
result, err = p.parseSimpleSelectorSequence()
|
||||
result, err := p.parseSimpleSelectorSequence()
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for {
|
||||
var combinator byte
|
||||
var (
|
||||
combinator byte
|
||||
c Sel
|
||||
)
|
||||
if p.skipWhitespace() {
|
||||
combinator = ' '
|
||||
}
|
||||
if p.i >= len(p.s) {
|
||||
return
|
||||
return result, nil
|
||||
}
|
||||
|
||||
switch p.s[p.i] {
|
||||
@@ -785,51 +763,39 @@ func (p *parser) parseSelector() (result Selector, err error) {
|
||||
p.skipWhitespace()
|
||||
case ',', ')':
|
||||
// These characters can't begin a selector, but they can legally occur after one.
|
||||
return
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if combinator == 0 {
|
||||
return
|
||||
return result, nil
|
||||
}
|
||||
|
||||
c, err := p.parseSimpleSelectorSequence()
|
||||
c, err = p.parseSimpleSelectorSequence()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch combinator {
|
||||
case ' ':
|
||||
result = descendantSelector(result, c)
|
||||
case '>':
|
||||
result = childSelector(result, c)
|
||||
case '+':
|
||||
result = siblingSelector(result, c, true)
|
||||
case '~':
|
||||
result = siblingSelector(result, c, false)
|
||||
}
|
||||
result = combinedSelector{first: result, combinator: combinator, second: c}
|
||||
}
|
||||
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
// parseSelectorGroup parses a group of selectors, separated by commas.
|
||||
func (p *parser) parseSelectorGroup() (result Selector, err error) {
|
||||
result, err = p.parseSelector()
|
||||
func (p *parser) parseSelectorGroup() (SelectorGroup, error) {
|
||||
current, err := p.parseSelector()
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
result := SelectorGroup{current}
|
||||
|
||||
for p.i < len(p.s) {
|
||||
if p.s[p.i] != ',' {
|
||||
return result, nil
|
||||
break
|
||||
}
|
||||
p.i++
|
||||
c, err := p.parseSelector()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = unionSelector(result, c)
|
||||
result = append(result, c)
|
||||
}
|
||||
|
||||
return
|
||||
return result, nil
|
||||
}
|
||||
|
||||
+600
-389
File diff suppressed because it is too large
Load Diff
+26
@@ -0,0 +1,26 @@
|
||||
package cascadia
|
||||
|
||||
// Specificity is the CSS specificity as defined in
|
||||
// https://www.w3.org/TR/selectors/#specificity-rules
|
||||
// with the convention Specificity = [A,B,C].
|
||||
type Specificity [3]int
|
||||
|
||||
// returns `true` if s < other (strictly), false otherwise
|
||||
func (s Specificity) Less(other Specificity) bool {
|
||||
for i := range s {
|
||||
if s[i] < other[i] {
|
||||
return true
|
||||
}
|
||||
if s[i] > other[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s Specificity) Add(other Specificity) Specificity {
|
||||
for i, sp := range other {
|
||||
s[i] += sp
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
_harness
|
||||
.vscode
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
language: go
|
||||
|
||||
go_import_path: github.com/globalsign/mgo
|
||||
|
||||
go:
|
||||
- 1.9.x
|
||||
- 1.10.x
|
||||
|
||||
env:
|
||||
global:
|
||||
- BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo
|
||||
- FASTDL=https://fastdl.mongodb.org/linux
|
||||
matrix:
|
||||
- MONGODB=x86_64-ubuntu1404-3.0.15
|
||||
- MONGODB=x86_64-ubuntu1404-3.2.17
|
||||
- MONGODB=x86_64-ubuntu1404-3.4.10
|
||||
- MONGODB=x86_64-ubuntu1404-3.6.0
|
||||
|
||||
install:
|
||||
|
||||
- wget $FASTDL/mongodb-linux-$MONGODB.tgz
|
||||
- tar xzvf mongodb-linux-$MONGODB.tgz
|
||||
- export PATH=$PWD/mongodb-linux-$MONGODB/bin:$PATH
|
||||
|
||||
- wget $BUCKET/daemontools.tar.gz
|
||||
- tar xzvf daemontools.tar.gz
|
||||
- export PATH=$PWD/daemontools:$PATH
|
||||
|
||||
- go get gopkg.in/check.v1
|
||||
- go get gopkg.in/yaml.v2
|
||||
- go get gopkg.in/tomb.v2
|
||||
- go get golang.org/x/lint/golint
|
||||
|
||||
before_script:
|
||||
- golint ./... | grep -v 'ID' | cat
|
||||
- go vet github.com/globalsign/mgo/bson github.com/globalsign/mgo/txn github.com/globalsign/mgo
|
||||
- export NOIPV6=1
|
||||
- make startdb
|
||||
|
||||
script:
|
||||
- (cd bson && go test -check.v)
|
||||
- go test -check.v -fast
|
||||
- (cd txn && go test -check.v)
|
||||
- make stopdb
|
||||
|
||||
git:
|
||||
depth: 3
|
||||
|
||||
# vim:sw=4:ts=4:et
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
Contributing
|
||||
-------------------------
|
||||
|
||||
We really appreciate contributions, but they must meet the following requirements:
|
||||
|
||||
* A PR should have a brief description of the problem/feature being proposed
|
||||
* Pull requests should target the `development` branch
|
||||
* Existing tests should pass and any new code should be covered with it's own test(s) (use [travis-ci](https://travis-ci.org))
|
||||
* New functions should be [documented](https://blog.golang.org/godoc-documenting-go-code) clearly
|
||||
* Code should pass `golint`, `go vet` and `go fmt`
|
||||
|
||||
We merge PRs into `development`, which is then tested in a sharded, replicated environment in our datacenter for regressions. Once everyone is happy, we merge to master - this is to maintain a bit of quality control past the usual PR process.
|
||||
|
||||
**Thanks** for helping!
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
mgo - MongoDB driver for Go
|
||||
|
||||
Copyright (c) 2010-2013 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
startdb:
|
||||
@harness/setup.sh start
|
||||
|
||||
stopdb:
|
||||
@harness/setup.sh stop
|
||||
-105
@@ -1,105 +0,0 @@
|
||||
[](https://travis-ci.org/globalsign/mgo) [](https://godoc.org/github.com/globalsign/mgo)
|
||||
|
||||
The MongoDB driver for Go
|
||||
-------------------------
|
||||
|
||||
This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review.
|
||||
Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added.
|
||||
|
||||
Further PR's (with tests) are welcome, but please maintain backwards compatibility.
|
||||
|
||||
Detailed documentation of the API is available at
|
||||
[GoDoc](https://godoc.org/github.com/globalsign/mgo).
|
||||
|
||||
A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver.
|
||||
|
||||
## Supported Versions
|
||||
|
||||
`mgo` is known to work well on (and has integration tests against) MongoDB v3.0, 3.2, 3.4 and 3.6.
|
||||
|
||||
MongoDB 4.0 is currently experimental - we would happily accept PRs to help improve support!
|
||||
|
||||
## Changes
|
||||
* Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254))
|
||||
* Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288))
|
||||
* Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373))
|
||||
* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56))
|
||||
* Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2))
|
||||
* Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5))
|
||||
* Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7))
|
||||
* Support for partial indexes ([details](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51))
|
||||
* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464))
|
||||
* Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35))
|
||||
* Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16))
|
||||
* Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899))
|
||||
* Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17))
|
||||
* Don't panic when handling indexed `int64` fields ([details](https://github.com/go-mgo/mgo/issues/475))
|
||||
* Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25))
|
||||
* Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28))
|
||||
* Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33))
|
||||
* Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37), [more](https://github.com/globalsign/mgo/pull/166))
|
||||
* Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41))
|
||||
* Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42))
|
||||
* Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44))
|
||||
* GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40))
|
||||
* Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49))
|
||||
* Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52))
|
||||
* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55))
|
||||
* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69))
|
||||
* Use JSON tags when no explicit BSON are tags set ([details](https://github.com/globalsign/mgo/pull/91))
|
||||
* Support [$changeStream](https://docs.mongodb.com/manual/changeStreams/) tailing on 3.6+ ([details](https://github.com/globalsign/mgo/pull/97))
|
||||
* Fix deadlock in cluster synchronisation ([details](https://github.com/globalsign/mgo/issues/120))
|
||||
* Implement `maxIdleTimeout` for pooled connections ([details](https://github.com/globalsign/mgo/pull/116))
|
||||
* Connection pool waiting improvements ([details](https://github.com/globalsign/mgo/pull/115))
|
||||
* Fixes BSON encoding for `$in` and friends ([details](https://github.com/globalsign/mgo/pull/128))
|
||||
* Add BSON stream encoders ([details](https://github.com/globalsign/mgo/pull/127))
|
||||
* Add integer map key support in the BSON encoder ([details](https://github.com/globalsign/mgo/pull/140))
|
||||
* Support aggregation [collations](https://docs.mongodb.com/manual/reference/collation/) ([details](https://github.com/globalsign/mgo/pull/144))
|
||||
* Support encoding of inline struct references ([details](https://github.com/globalsign/mgo/pull/146))
|
||||
* Improved windows test harness ([details](https://github.com/globalsign/mgo/pull/158))
|
||||
* Improved type and nil handling in the BSON codec ([details](https://github.com/globalsign/mgo/pull/147/files), [more](https://github.com/globalsign/mgo/pull/181))
|
||||
* Separated network read/write timeouts ([details](https://github.com/globalsign/mgo/pull/161))
|
||||
* Expanded dial string configuration options ([details](https://github.com/globalsign/mgo/pull/162))
|
||||
* Implement MongoTimestamp ([details](https://github.com/globalsign/mgo/pull/171))
|
||||
* Support setting `writeConcern` for `findAndModify` operations ([details](https://github.com/globalsign/mgo/pull/185))
|
||||
* Add `ssl` to the dial string options ([details](https://github.com/globalsign/mgo/pull/184))
|
||||
|
||||
|
||||
---
|
||||
|
||||
### Thanks to
|
||||
* @aksentyev
|
||||
* @bachue
|
||||
* @bozaro
|
||||
* @BenLubar
|
||||
* @carldunham
|
||||
* @carter2000
|
||||
* @cedric-cordenier
|
||||
* @cezarsa
|
||||
* @DaytonG
|
||||
* @ddspog
|
||||
* @drichelson
|
||||
* @dvic
|
||||
* @eaglerayp
|
||||
* @feliixx
|
||||
* @fmpwizard
|
||||
* @gazoon
|
||||
* @gedge
|
||||
* @gnawux
|
||||
* @idy
|
||||
* @jameinel
|
||||
* @jefferickson
|
||||
* @johnlawsharrison
|
||||
* @KJTsanaktsidis
|
||||
* @larrycinnabar
|
||||
* @mapete94
|
||||
* @maxnoel
|
||||
* @mcspring
|
||||
* @Mei-Zhao
|
||||
* @peterdeka
|
||||
* @Reenjii
|
||||
* @roobre
|
||||
* @smoya
|
||||
* @steve-gray
|
||||
* @tbruyelle
|
||||
* @wgallagher
|
||||
-467
@@ -1,467 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"github.com/globalsign/mgo/internal/scram"
|
||||
)
|
||||
|
||||
type authCmd struct {
|
||||
Authenticate int
|
||||
|
||||
Nonce string
|
||||
User string
|
||||
Key string
|
||||
}
|
||||
|
||||
type startSaslCmd struct {
|
||||
StartSASL int `bson:"startSasl"`
|
||||
}
|
||||
|
||||
type authResult struct {
|
||||
ErrMsg string
|
||||
Ok bool
|
||||
}
|
||||
|
||||
type getNonceCmd struct {
|
||||
GetNonce int
|
||||
}
|
||||
|
||||
type getNonceResult struct {
|
||||
Nonce string
|
||||
Err string `bson:"$err"`
|
||||
Code int
|
||||
}
|
||||
|
||||
type logoutCmd struct {
|
||||
Logout int
|
||||
}
|
||||
|
||||
type saslCmd struct {
|
||||
Start int `bson:"saslStart,omitempty"`
|
||||
Continue int `bson:"saslContinue,omitempty"`
|
||||
ConversationId int `bson:"conversationId,omitempty"`
|
||||
Mechanism string `bson:"mechanism,omitempty"`
|
||||
Payload []byte
|
||||
}
|
||||
|
||||
type saslResult struct {
|
||||
Ok bool `bson:"ok"`
|
||||
NotOk bool `bson:"code"` // Server <= 2.3.2 returns ok=1 & code>0 on errors (WTF?)
|
||||
Done bool
|
||||
|
||||
ConversationId int `bson:"conversationId"`
|
||||
Payload []byte
|
||||
ErrMsg string
|
||||
}
|
||||
|
||||
type saslStepper interface {
|
||||
Step(serverData []byte) (clientData []byte, done bool, err error)
|
||||
Close()
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) getNonce() (nonce string, err error) {
|
||||
socket.Lock()
|
||||
for socket.cachedNonce == "" && socket.dead == nil {
|
||||
debugf("Socket %p to %s: waiting for nonce", socket, socket.addr)
|
||||
socket.gotNonce.Wait()
|
||||
}
|
||||
if socket.cachedNonce == "mongos" {
|
||||
socket.Unlock()
|
||||
return "", errors.New("Can't authenticate with mongos; see http://j.mp/mongos-auth")
|
||||
}
|
||||
debugf("Socket %p to %s: got nonce", socket, socket.addr)
|
||||
nonce, err = socket.cachedNonce, socket.dead
|
||||
socket.cachedNonce = ""
|
||||
socket.Unlock()
|
||||
if err != nil {
|
||||
nonce = ""
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) resetNonce() {
|
||||
debugf("Socket %p to %s: requesting a new nonce", socket, socket.addr)
|
||||
op := &queryOp{}
|
||||
op.query = &getNonceCmd{GetNonce: 1}
|
||||
op.collection = "admin.$cmd"
|
||||
op.limit = -1
|
||||
op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) {
|
||||
if err != nil {
|
||||
socket.kill(errors.New("getNonce: "+err.Error()), true)
|
||||
return
|
||||
}
|
||||
result := &getNonceResult{}
|
||||
err = bson.Unmarshal(docData, &result)
|
||||
if err != nil {
|
||||
socket.kill(errors.New("Failed to unmarshal nonce: "+err.Error()), true)
|
||||
return
|
||||
}
|
||||
debugf("Socket %p to %s: nonce unmarshalled: %#v", socket, socket.addr, result)
|
||||
if result.Code == 13390 {
|
||||
// mongos doesn't yet support auth (see http://j.mp/mongos-auth)
|
||||
result.Nonce = "mongos"
|
||||
} else if result.Nonce == "" {
|
||||
var msg string
|
||||
if result.Err != "" {
|
||||
msg = fmt.Sprintf("Got an empty nonce: %s (%d)", result.Err, result.Code)
|
||||
} else {
|
||||
msg = "Got an empty nonce"
|
||||
}
|
||||
socket.kill(errors.New(msg), true)
|
||||
return
|
||||
}
|
||||
socket.Lock()
|
||||
if socket.cachedNonce != "" {
|
||||
socket.Unlock()
|
||||
panic("resetNonce: nonce already cached")
|
||||
}
|
||||
socket.cachedNonce = result.Nonce
|
||||
socket.gotNonce.Signal()
|
||||
socket.Unlock()
|
||||
}
|
||||
err := socket.Query(op)
|
||||
if err != nil {
|
||||
socket.kill(errors.New("resetNonce: "+err.Error()), true)
|
||||
}
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) Login(cred Credential) error {
|
||||
socket.Lock()
|
||||
if cred.Mechanism == "" && socket.serverInfo.MaxWireVersion >= 3 {
|
||||
cred.Mechanism = "SCRAM-SHA-1"
|
||||
}
|
||||
for _, sockCred := range socket.creds {
|
||||
if sockCred == cred {
|
||||
debugf("Socket %p to %s: login: db=%q user=%q (already logged in)", socket, socket.addr, cred.Source, cred.Username)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if socket.dropLogout(cred) {
|
||||
debugf("Socket %p to %s: login: db=%q user=%q (cached)", socket, socket.addr, cred.Source, cred.Username)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
}
|
||||
socket.Unlock()
|
||||
|
||||
debugf("Socket %p to %s: login: db=%q user=%q", socket, socket.addr, cred.Source, cred.Username)
|
||||
|
||||
var err error
|
||||
switch cred.Mechanism {
|
||||
case "", "MONGODB-CR", "MONGO-CR": // Name changed to MONGODB-CR in SERVER-8501.
|
||||
err = socket.loginClassic(cred)
|
||||
case "PLAIN":
|
||||
err = socket.loginPlain(cred)
|
||||
case "MONGODB-X509":
|
||||
err = socket.loginX509(cred)
|
||||
default:
|
||||
// Try SASL for everything else, if it is available.
|
||||
err = socket.loginSASL(cred)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
debugf("Socket %p to %s: login error: %s", socket, socket.addr, err)
|
||||
} else {
|
||||
debugf("Socket %p to %s: login successful", socket, socket.addr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) loginClassic(cred Credential) error {
|
||||
// Note that this only works properly because this function is
|
||||
// synchronous, which means the nonce won't get reset while we're
|
||||
// using it and any other login requests will block waiting for a
|
||||
// new nonce provided in the defer call below.
|
||||
nonce, err := socket.getNonce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer socket.resetNonce()
|
||||
|
||||
psum := md5.New()
|
||||
psum.Write([]byte(cred.Username + ":mongo:" + cred.Password))
|
||||
|
||||
ksum := md5.New()
|
||||
ksum.Write([]byte(nonce + cred.Username))
|
||||
ksum.Write([]byte(hex.EncodeToString(psum.Sum(nil))))
|
||||
|
||||
key := hex.EncodeToString(ksum.Sum(nil))
|
||||
|
||||
cmd := authCmd{Authenticate: 1, User: cred.Username, Nonce: nonce, Key: key}
|
||||
res := authResult{}
|
||||
return socket.loginRun(cred.Source, &cmd, &res, func() error {
|
||||
if !res.Ok {
|
||||
return errors.New(res.ErrMsg)
|
||||
}
|
||||
socket.Lock()
|
||||
socket.dropAuth(cred.Source)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
type authX509Cmd struct {
|
||||
Authenticate int
|
||||
User string
|
||||
Mechanism string
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) loginX509(cred Credential) error {
|
||||
cmd := authX509Cmd{Authenticate: 1, User: cred.Username, Mechanism: "MONGODB-X509"}
|
||||
res := authResult{}
|
||||
return socket.loginRun(cred.Source, &cmd, &res, func() error {
|
||||
if !res.Ok {
|
||||
return errors.New(res.ErrMsg)
|
||||
}
|
||||
socket.Lock()
|
||||
socket.dropAuth(cred.Source)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) loginPlain(cred Credential) error {
|
||||
cmd := saslCmd{Start: 1, Mechanism: "PLAIN", Payload: []byte("\x00" + cred.Username + "\x00" + cred.Password)}
|
||||
res := authResult{}
|
||||
return socket.loginRun(cred.Source, &cmd, &res, func() error {
|
||||
if !res.Ok {
|
||||
return errors.New(res.ErrMsg)
|
||||
}
|
||||
socket.Lock()
|
||||
socket.dropAuth(cred.Source)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) loginSASL(cred Credential) error {
|
||||
var sasl saslStepper
|
||||
var err error
|
||||
if cred.Mechanism == "SCRAM-SHA-1" {
|
||||
// SCRAM is handled without external libraries.
|
||||
sasl = saslNewScram(cred)
|
||||
} else if len(cred.ServiceHost) > 0 {
|
||||
sasl, err = saslNew(cred, cred.ServiceHost)
|
||||
} else {
|
||||
sasl, err = saslNew(cred, socket.Server().Addr)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sasl.Close()
|
||||
|
||||
// The goal of this logic is to carry a locked socket until the
|
||||
// local SASL step confirms the auth is valid; the socket needs to be
|
||||
// locked so that concurrent action doesn't leave the socket in an
|
||||
// auth state that doesn't reflect the operations that took place.
|
||||
// As a simple case, imagine inverting login=>logout to logout=>login.
|
||||
//
|
||||
// The logic below works because the lock func isn't called concurrently.
|
||||
locked := false
|
||||
lock := func(b bool) {
|
||||
if locked != b {
|
||||
locked = b
|
||||
if b {
|
||||
socket.Lock()
|
||||
} else {
|
||||
socket.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lock(true)
|
||||
defer lock(false)
|
||||
|
||||
start := 1
|
||||
cmd := saslCmd{}
|
||||
res := saslResult{}
|
||||
for {
|
||||
payload, done, err := sasl.Step(res.Payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done && res.Done {
|
||||
socket.dropAuth(cred.Source)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
break
|
||||
}
|
||||
lock(false)
|
||||
|
||||
cmd = saslCmd{
|
||||
Start: start,
|
||||
Continue: 1 - start,
|
||||
ConversationId: res.ConversationId,
|
||||
Mechanism: cred.Mechanism,
|
||||
Payload: payload,
|
||||
}
|
||||
start = 0
|
||||
err = socket.loginRun(cred.Source, &cmd, &res, func() error {
|
||||
// See the comment on lock for why this is necessary.
|
||||
lock(true)
|
||||
if !res.Ok || res.NotOk {
|
||||
return fmt.Errorf("server returned error on SASL authentication step: %s", res.ErrMsg)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done && res.Done {
|
||||
socket.dropAuth(cred.Source)
|
||||
socket.creds = append(socket.creds, cred)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func saslNewScram(cred Credential) *saslScram {
|
||||
credsum := md5.New()
|
||||
credsum.Write([]byte(cred.Username + ":mongo:" + cred.Password))
|
||||
client := scram.NewClient(sha1.New, cred.Username, hex.EncodeToString(credsum.Sum(nil)))
|
||||
return &saslScram{cred: cred, client: client}
|
||||
}
|
||||
|
||||
type saslScram struct {
|
||||
cred Credential
|
||||
client *scram.Client
|
||||
}
|
||||
|
||||
func (s *saslScram) Close() {}
|
||||
|
||||
func (s *saslScram) Step(serverData []byte) (clientData []byte, done bool, err error) {
|
||||
more := s.client.Step(serverData)
|
||||
return s.client.Out(), !more, s.client.Err()
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) loginRun(db string, query, result interface{}, f func() error) error {
|
||||
var mutex sync.Mutex
|
||||
var replyErr error
|
||||
mutex.Lock()
|
||||
|
||||
op := queryOp{}
|
||||
op.query = query
|
||||
op.collection = db + ".$cmd"
|
||||
op.limit = -1
|
||||
op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) {
|
||||
defer mutex.Unlock()
|
||||
|
||||
if err != nil {
|
||||
replyErr = err
|
||||
return
|
||||
}
|
||||
|
||||
err = bson.Unmarshal(docData, result)
|
||||
if err != nil {
|
||||
replyErr = err
|
||||
} else {
|
||||
// Must handle this within the read loop for the socket, so
|
||||
// that concurrent login requests are properly ordered.
|
||||
replyErr = f()
|
||||
}
|
||||
}
|
||||
|
||||
err := socket.Query(&op)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mutex.Lock() // Wait.
|
||||
return replyErr
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) Logout(db string) {
|
||||
socket.Lock()
|
||||
cred, found := socket.dropAuth(db)
|
||||
if found {
|
||||
debugf("Socket %p to %s: logout: db=%q (flagged)", socket, socket.addr, db)
|
||||
socket.logout = append(socket.logout, cred)
|
||||
}
|
||||
socket.Unlock()
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) LogoutAll() {
|
||||
socket.Lock()
|
||||
if l := len(socket.creds); l > 0 {
|
||||
debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l)
|
||||
socket.logout = append(socket.logout, socket.creds...)
|
||||
socket.creds = socket.creds[0:0]
|
||||
}
|
||||
socket.Unlock()
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) flushLogout() (ops []interface{}) {
|
||||
socket.Lock()
|
||||
if l := len(socket.logout); l > 0 {
|
||||
debugf("Socket %p to %s: logout all (flushing %d)", socket, socket.addr, l)
|
||||
for i := 0; i != l; i++ {
|
||||
op := queryOp{}
|
||||
op.query = &logoutCmd{1}
|
||||
op.collection = socket.logout[i].Source + ".$cmd"
|
||||
op.limit = -1
|
||||
ops = append(ops, &op)
|
||||
}
|
||||
socket.logout = socket.logout[0:0]
|
||||
}
|
||||
socket.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) {
|
||||
for i, sockCred := range socket.creds {
|
||||
if sockCred.Source == db {
|
||||
copy(socket.creds[i:], socket.creds[i+1:])
|
||||
socket.creds = socket.creds[:len(socket.creds)-1]
|
||||
return sockCred, true
|
||||
}
|
||||
}
|
||||
return cred, false
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) dropLogout(cred Credential) (found bool) {
|
||||
for i, sockCred := range socket.logout {
|
||||
if sockCred == cred {
|
||||
copy(socket.logout[i:], socket.logout[i+1:])
|
||||
socket.logout = socket.logout[:len(socket.logout)-1]
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
BSON library for Go
|
||||
|
||||
Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
[](https://godoc.org/github.com/globalsign/mgo/bson)
|
||||
|
||||
An Implementation of BSON for Go
|
||||
--------------------------------
|
||||
|
||||
Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go.
|
||||
|
||||
While the BSON package implements the BSON spec as faithfully as possible, there
|
||||
is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the
|
||||
`bson` package. The priority is for backwards compatibility for the `mgo`
|
||||
driver, though fixes for obviously buggy behaviour is welcome (and features, etc
|
||||
behind feature flags).
|
||||
-836
@@ -1,836 +0,0 @@
|
||||
// BSON library for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Package bson is an implementation of the BSON specification for Go:
|
||||
//
|
||||
// http://bsonspec.org
|
||||
//
|
||||
// It was created as part of the mgo MongoDB driver for Go, but is standalone
|
||||
// and may be used on its own without the driver.
|
||||
package bson
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:generate go run bson_corpus_spec_test_generator.go
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// The public API.
|
||||
|
||||
// Element types constants from BSON specification.
|
||||
const (
|
||||
ElementFloat64 byte = 0x01
|
||||
ElementString byte = 0x02
|
||||
ElementDocument byte = 0x03
|
||||
ElementArray byte = 0x04
|
||||
ElementBinary byte = 0x05
|
||||
Element06 byte = 0x06
|
||||
ElementObjectId byte = 0x07
|
||||
ElementBool byte = 0x08
|
||||
ElementDatetime byte = 0x09
|
||||
ElementNil byte = 0x0A
|
||||
ElementRegEx byte = 0x0B
|
||||
ElementDBPointer byte = 0x0C
|
||||
ElementJavaScriptWithoutScope byte = 0x0D
|
||||
ElementSymbol byte = 0x0E
|
||||
ElementJavaScriptWithScope byte = 0x0F
|
||||
ElementInt32 byte = 0x10
|
||||
ElementTimestamp byte = 0x11
|
||||
ElementInt64 byte = 0x12
|
||||
ElementDecimal128 byte = 0x13
|
||||
ElementMinKey byte = 0xFF
|
||||
ElementMaxKey byte = 0x7F
|
||||
|
||||
BinaryGeneric byte = 0x00
|
||||
BinaryFunction byte = 0x01
|
||||
BinaryBinaryOld byte = 0x02
|
||||
BinaryUUIDOld byte = 0x03
|
||||
BinaryUUID byte = 0x04
|
||||
BinaryMD5 byte = 0x05
|
||||
BinaryUserDefined byte = 0x80
|
||||
)
|
||||
|
||||
// Getter interface: a value implementing the bson.Getter interface will have its GetBSON
|
||||
// method called when the given value has to be marshalled, and the result
|
||||
// of this method will be marshaled in place of the actual object.
|
||||
//
|
||||
// If GetBSON returns return a non-nil error, the marshalling procedure
|
||||
// will stop and error out with the provided value.
|
||||
type Getter interface {
|
||||
GetBSON() (interface{}, error)
|
||||
}
|
||||
|
||||
// Setter interface: a value implementing the bson.Setter interface will receive the BSON
|
||||
// value via the SetBSON method during unmarshaling, and the object
|
||||
// itself will not be changed as usual.
|
||||
//
|
||||
// If setting the value works, the method should return nil or alternatively
|
||||
// bson.ErrSetZero to set the respective field to its zero value (nil for
|
||||
// pointer types). If SetBSON returns a value of type bson.TypeError, the
|
||||
// BSON value will be omitted from a map or slice being decoded and the
|
||||
// unmarshalling will continue. If it returns any other non-nil error, the
|
||||
// unmarshalling procedure will stop and error out with the provided value.
|
||||
//
|
||||
// This interface is generally useful in pointer receivers, since the method
|
||||
// will want to change the receiver. A type field that implements the Setter
|
||||
// interface doesn't have to be a pointer, though.
|
||||
//
|
||||
// Unlike the usual behavior, unmarshalling onto a value that implements a
|
||||
// Setter interface will NOT reset the value to its zero state. This allows
|
||||
// the value to decide by itself how to be unmarshalled.
|
||||
//
|
||||
// For example:
|
||||
//
|
||||
// type MyString string
|
||||
//
|
||||
// func (s *MyString) SetBSON(raw bson.Raw) error {
|
||||
// return raw.Unmarshal(s)
|
||||
// }
|
||||
//
|
||||
type Setter interface {
|
||||
SetBSON(raw Raw) error
|
||||
}
|
||||
|
||||
// ErrSetZero may be returned from a SetBSON method to have the value set to
|
||||
// its respective zero value. When used in pointer values, this will set the
|
||||
// field to nil rather than to the pre-allocated value.
|
||||
var ErrSetZero = errors.New("set to zero")
|
||||
|
||||
// M is a convenient alias for a map[string]interface{} map, useful for
|
||||
// dealing with BSON in a native way. For instance:
|
||||
//
|
||||
// bson.M{"a": 1, "b": true}
|
||||
//
|
||||
// There's no special handling for this type in addition to what's done anyway
|
||||
// for an equivalent map type. Elements in the map will be dumped in an
|
||||
// undefined ordered. See also the bson.D type for an ordered alternative.
|
||||
type M map[string]interface{}
|
||||
|
||||
// D represents a BSON document containing ordered elements. For example:
|
||||
//
|
||||
// bson.D{{"a", 1}, {"b", true}}
|
||||
//
|
||||
// In some situations, such as when creating indexes for MongoDB, the order in
|
||||
// which the elements are defined is important. If the order is not important,
|
||||
// using a map is generally more comfortable. See bson.M and bson.RawD.
|
||||
type D []DocElem
|
||||
|
||||
// DocElem is an element of the bson.D document representation.
|
||||
type DocElem struct {
|
||||
Name string
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// Map returns a map out of the ordered element name/value pairs in d.
|
||||
func (d D) Map() (m M) {
|
||||
m = make(M, len(d))
|
||||
for _, item := range d {
|
||||
m[item.Name] = item.Value
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The Raw type represents raw unprocessed BSON documents and elements.
|
||||
// Kind is the kind of element as defined per the BSON specification, and
|
||||
// Data is the raw unprocessed data for the respective element.
|
||||
// Using this type it is possible to unmarshal or marshal values partially.
|
||||
//
|
||||
// Relevant documentation:
|
||||
//
|
||||
// http://bsonspec.org/#/specification
|
||||
//
|
||||
type Raw struct {
|
||||
Kind byte
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// RawD represents a BSON document containing raw unprocessed elements.
|
||||
// This low-level representation may be useful when lazily processing
|
||||
// documents of uncertain content, or when manipulating the raw content
|
||||
// documents in general.
|
||||
type RawD []RawDocElem
|
||||
|
||||
// RawDocElem elements of RawD type.
|
||||
type RawDocElem struct {
|
||||
Name string
|
||||
Value Raw
|
||||
}
|
||||
|
||||
// ObjectId is a unique ID identifying a BSON value. It must be exactly 12 bytes
|
||||
// long. MongoDB objects by default have such a property set in their "_id"
|
||||
// property.
|
||||
//
|
||||
// http://www.mongodb.org/display/DOCS/Object+Ids
|
||||
type ObjectId string
|
||||
|
||||
// ObjectIdHex returns an ObjectId from the provided hex representation.
|
||||
// Calling this function with an invalid hex representation will
|
||||
// cause a runtime panic. See the IsObjectIdHex function.
|
||||
func ObjectIdHex(s string) ObjectId {
|
||||
d, err := hex.DecodeString(s)
|
||||
if err != nil || len(d) != 12 {
|
||||
panic(fmt.Sprintf("invalid input to ObjectIdHex: %q", s))
|
||||
}
|
||||
return ObjectId(d)
|
||||
}
|
||||
|
||||
// IsObjectIdHex returns whether s is a valid hex representation of
|
||||
// an ObjectId. See the ObjectIdHex function.
|
||||
func IsObjectIdHex(s string) bool {
|
||||
if len(s) != 24 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// objectIdCounter is atomically incremented when generating a new ObjectId
|
||||
// using NewObjectId() function. It's used as a counter part of an id.
|
||||
var objectIdCounter = readRandomUint32()
|
||||
|
||||
// readRandomUint32 returns a random objectIdCounter.
|
||||
func readRandomUint32() uint32 {
|
||||
var b [4]byte
|
||||
_, err := io.ReadFull(rand.Reader, b[:])
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot read random object id: %v", err))
|
||||
}
|
||||
return uint32((uint32(b[0]) << 0) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24))
|
||||
}
|
||||
|
||||
// machineId stores machine id generated once and used in subsequent calls
|
||||
// to NewObjectId function.
|
||||
var machineId = readMachineId()
|
||||
var processId = os.Getpid()
|
||||
|
||||
// readMachineId generates and returns a machine id.
|
||||
// If this function fails to get the hostname it will cause a runtime error.
|
||||
func readMachineId() []byte {
|
||||
var sum [3]byte
|
||||
id := sum[:]
|
||||
hostname, err1 := os.Hostname()
|
||||
if err1 != nil {
|
||||
_, err2 := io.ReadFull(rand.Reader, id)
|
||||
if err2 != nil {
|
||||
panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2))
|
||||
}
|
||||
return id
|
||||
}
|
||||
hw := md5.New()
|
||||
hw.Write([]byte(hostname))
|
||||
copy(id, hw.Sum(nil))
|
||||
return id
|
||||
}
|
||||
|
||||
// NewObjectId returns a new unique ObjectId.
|
||||
func NewObjectId() ObjectId {
|
||||
var b [12]byte
|
||||
// Timestamp, 4 bytes, big endian
|
||||
binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix()))
|
||||
// Machine, first 3 bytes of md5(hostname)
|
||||
b[4] = machineId[0]
|
||||
b[5] = machineId[1]
|
||||
b[6] = machineId[2]
|
||||
// Pid, 2 bytes, specs don't specify endianness, but we use big endian.
|
||||
b[7] = byte(processId >> 8)
|
||||
b[8] = byte(processId)
|
||||
// Increment, 3 bytes, big endian
|
||||
i := atomic.AddUint32(&objectIdCounter, 1)
|
||||
b[9] = byte(i >> 16)
|
||||
b[10] = byte(i >> 8)
|
||||
b[11] = byte(i)
|
||||
return ObjectId(b[:])
|
||||
}
|
||||
|
||||
// NewObjectIdWithTime returns a dummy ObjectId with the timestamp part filled
|
||||
// with the provided number of seconds from epoch UTC, and all other parts
|
||||
// filled with zeroes. It's not safe to insert a document with an id generated
|
||||
// by this method, it is useful only for queries to find documents with ids
|
||||
// generated before or after the specified timestamp.
|
||||
func NewObjectIdWithTime(t time.Time) ObjectId {
|
||||
var b [12]byte
|
||||
binary.BigEndian.PutUint32(b[:4], uint32(t.Unix()))
|
||||
return ObjectId(string(b[:]))
|
||||
}
|
||||
|
||||
// String returns a hex string representation of the id.
|
||||
// Example: ObjectIdHex("4d88e15b60f486e428412dc9").
|
||||
func (id ObjectId) String() string {
|
||||
return fmt.Sprintf(`ObjectIdHex("%x")`, string(id))
|
||||
}
|
||||
|
||||
// Hex returns a hex representation of the ObjectId.
|
||||
func (id ObjectId) Hex() string {
|
||||
return hex.EncodeToString([]byte(id))
|
||||
}
|
||||
|
||||
// MarshalJSON turns a bson.ObjectId into a json.Marshaller.
|
||||
func (id ObjectId) MarshalJSON() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf(`"%x"`, string(id))), nil
|
||||
}
|
||||
|
||||
var nullBytes = []byte("null")
|
||||
|
||||
// UnmarshalJSON turns *bson.ObjectId into a json.Unmarshaller.
|
||||
func (id *ObjectId) UnmarshalJSON(data []byte) error {
|
||||
if len(data) > 0 && (data[0] == '{' || data[0] == 'O') {
|
||||
var v struct {
|
||||
Id json.RawMessage `json:"$oid"`
|
||||
Func struct {
|
||||
Id json.RawMessage
|
||||
} `json:"$oidFunc"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err == nil {
|
||||
if len(v.Id) > 0 {
|
||||
data = []byte(v.Id)
|
||||
} else {
|
||||
data = []byte(v.Func.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(data) == 2 && data[0] == '"' && data[1] == '"' || bytes.Equal(data, nullBytes) {
|
||||
*id = ""
|
||||
return nil
|
||||
}
|
||||
if len(data) != 26 || data[0] != '"' || data[25] != '"' {
|
||||
return fmt.Errorf("invalid ObjectId in JSON: %s", string(data))
|
||||
}
|
||||
var buf [12]byte
|
||||
_, err := hex.Decode(buf[:], data[1:25])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err)
|
||||
}
|
||||
*id = ObjectId(string(buf[:]))
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalText turns bson.ObjectId into an encoding.TextMarshaler.
|
||||
func (id ObjectId) MarshalText() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("%x", string(id))), nil
|
||||
}
|
||||
|
||||
// UnmarshalText turns *bson.ObjectId into an encoding.TextUnmarshaler.
|
||||
func (id *ObjectId) UnmarshalText(data []byte) error {
|
||||
if len(data) == 1 && data[0] == ' ' || len(data) == 0 {
|
||||
*id = ""
|
||||
return nil
|
||||
}
|
||||
if len(data) != 24 {
|
||||
return fmt.Errorf("invalid ObjectId: %s", data)
|
||||
}
|
||||
var buf [12]byte
|
||||
_, err := hex.Decode(buf[:], data[:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid ObjectId: %s (%s)", data, err)
|
||||
}
|
||||
*id = ObjectId(string(buf[:]))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Valid returns true if id is valid. A valid id must contain exactly 12 bytes.
|
||||
func (id ObjectId) Valid() bool {
|
||||
return len(id) == 12
|
||||
}
|
||||
|
||||
// byteSlice returns byte slice of id from start to end.
|
||||
// Calling this function with an invalid id will cause a runtime panic.
|
||||
func (id ObjectId) byteSlice(start, end int) []byte {
|
||||
if len(id) != 12 {
|
||||
panic(fmt.Sprintf("invalid ObjectId: %q", string(id)))
|
||||
}
|
||||
return []byte(string(id)[start:end])
|
||||
}
|
||||
|
||||
// Time returns the timestamp part of the id.
|
||||
// It's a runtime error to call this method with an invalid id.
|
||||
func (id ObjectId) Time() time.Time {
|
||||
// First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
|
||||
secs := int64(binary.BigEndian.Uint32(id.byteSlice(0, 4)))
|
||||
return time.Unix(secs, 0)
|
||||
}
|
||||
|
||||
// Machine returns the 3-byte machine id part of the id.
|
||||
// It's a runtime error to call this method with an invalid id.
|
||||
func (id ObjectId) Machine() []byte {
|
||||
return id.byteSlice(4, 7)
|
||||
}
|
||||
|
||||
// Pid returns the process id part of the id.
|
||||
// It's a runtime error to call this method with an invalid id.
|
||||
func (id ObjectId) Pid() uint16 {
|
||||
return binary.BigEndian.Uint16(id.byteSlice(7, 9))
|
||||
}
|
||||
|
||||
// Counter returns the incrementing value part of the id.
|
||||
// It's a runtime error to call this method with an invalid id.
|
||||
func (id ObjectId) Counter() int32 {
|
||||
b := id.byteSlice(9, 12)
|
||||
// Counter is stored as big-endian 3-byte value
|
||||
return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
|
||||
}
|
||||
|
||||
// The Symbol type is similar to a string and is used in languages with a
|
||||
// distinct symbol type.
|
||||
type Symbol string
|
||||
|
||||
// Now returns the current time with millisecond precision. MongoDB stores
|
||||
// timestamps with the same precision, so a Time returned from this method
|
||||
// will not change after a roundtrip to the database. That's the only reason
|
||||
// why this function exists. Using the time.Now function also works fine
|
||||
// otherwise.
|
||||
func Now() time.Time {
|
||||
return time.Unix(0, time.Now().UnixNano()/1e6*1e6)
|
||||
}
|
||||
|
||||
// MongoTimestamp is a special internal type used by MongoDB that for some
|
||||
// strange reason has its own datatype defined in BSON.
|
||||
type MongoTimestamp int64
|
||||
|
||||
// Time returns the time part of ts which is stored with second precision.
|
||||
func (ts MongoTimestamp) Time() time.Time {
|
||||
return time.Unix(int64(uint64(ts)>>32), 0)
|
||||
}
|
||||
|
||||
// Counter returns the counter part of ts.
|
||||
func (ts MongoTimestamp) Counter() uint32 {
|
||||
return uint32(ts)
|
||||
}
|
||||
|
||||
// NewMongoTimestamp creates a timestamp using the given
|
||||
// date `t` (with second precision) and counter `c` (unique for `t`).
|
||||
//
|
||||
// Returns an error if time `t` is not between 1970-01-01T00:00:00Z
|
||||
// and 2106-02-07T06:28:15Z (inclusive).
|
||||
//
|
||||
// Note that two MongoTimestamps should never have the same (time, counter) combination:
|
||||
// the caller must ensure the counter `c` is increased if creating multiple MongoTimestamp
|
||||
// values for the same time `t` (ignoring fractions of seconds).
|
||||
func NewMongoTimestamp(t time.Time, c uint32) (MongoTimestamp, error) {
|
||||
u := t.Unix()
|
||||
if u < 0 || u > math.MaxUint32 {
|
||||
return -1, errors.New("invalid value for time")
|
||||
}
|
||||
|
||||
i := int64(u<<32 | int64(c))
|
||||
|
||||
return MongoTimestamp(i), nil
|
||||
}
|
||||
|
||||
type orderKey int64
|
||||
|
||||
// MaxKey is a special value that compares higher than all other possible BSON
|
||||
// values in a MongoDB database.
|
||||
var MaxKey = orderKey(1<<63 - 1)
|
||||
|
||||
// MinKey is a special value that compares lower than all other possible BSON
|
||||
// values in a MongoDB database.
|
||||
var MinKey = orderKey(-1 << 63)
|
||||
|
||||
type undefined struct{}
|
||||
|
||||
// Undefined represents the undefined BSON value.
|
||||
var Undefined undefined
|
||||
|
||||
// Binary is a representation for non-standard binary values. Any kind should
|
||||
// work, but the following are known as of this writing:
|
||||
//
|
||||
// 0x00 - Generic. This is decoded as []byte(data), not Binary{0x00, data}.
|
||||
// 0x01 - Function (!?)
|
||||
// 0x02 - Obsolete generic.
|
||||
// 0x03 - UUID
|
||||
// 0x05 - MD5
|
||||
// 0x80 - User defined.
|
||||
//
|
||||
type Binary struct {
|
||||
Kind byte
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// RegEx represents a regular expression. The Options field may contain
|
||||
// individual characters defining the way in which the pattern should be
|
||||
// applied, and must be sorted. Valid options as of this writing are 'i' for
|
||||
// case insensitive matching, 'm' for multi-line matching, 'x' for verbose
|
||||
// mode, 'l' to make \w, \W, and similar be locale-dependent, 's' for dot-all
|
||||
// mode (a '.' matches everything), and 'u' to make \w, \W, and similar match
|
||||
// unicode. The value of the Options parameter is not verified before being
|
||||
// marshaled into the BSON format.
|
||||
type RegEx struct {
|
||||
Pattern string
|
||||
Options string
|
||||
}
|
||||
|
||||
// JavaScript is a type that holds JavaScript code. If Scope is non-nil, it
|
||||
// will be marshaled as a mapping from identifiers to values that may be
|
||||
// used when evaluating the provided Code.
|
||||
type JavaScript struct {
|
||||
Code string
|
||||
Scope interface{}
|
||||
}
|
||||
|
||||
// DBPointer refers to a document id in a namespace.
|
||||
//
|
||||
// This type is deprecated in the BSON specification and should not be used
|
||||
// except for backwards compatibility with ancient applications.
|
||||
type DBPointer struct {
|
||||
Namespace string
|
||||
Id ObjectId
|
||||
}
|
||||
|
||||
const initialBufferSize = 64
|
||||
|
||||
func handleErr(err *error) {
|
||||
if r := recover(); r != nil {
|
||||
if _, ok := r.(runtime.Error); ok {
|
||||
panic(r)
|
||||
} else if _, ok := r.(externalPanic); ok {
|
||||
panic(r)
|
||||
} else if s, ok := r.(string); ok {
|
||||
*err = errors.New(s)
|
||||
} else if e, ok := r.(error); ok {
|
||||
*err = e
|
||||
} else {
|
||||
panic(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Marshal serializes the in value, which may be a map or a struct value.
|
||||
// In the case of struct values, only exported fields will be serialized,
|
||||
// and the order of serialized fields will match that of the struct itself.
|
||||
// The lowercased field name is used as the key for each exported field,
|
||||
// but this behavior may be changed using the respective field tag.
|
||||
// The tag may also contain flags to tweak the marshalling behavior for
|
||||
// the field. The tag formats accepted are:
|
||||
//
|
||||
// "[<key>][,<flag1>[,<flag2>]]"
|
||||
//
|
||||
// `(...) bson:"[<key>][,<flag1>[,<flag2>]]" (...)`
|
||||
//
|
||||
// The following flags are currently supported:
|
||||
//
|
||||
// omitempty Only include the field if it's not set to the zero
|
||||
// value for the type or to empty slices or maps.
|
||||
//
|
||||
// minsize Marshal an int64 value as an int32, if that's feasible
|
||||
// while preserving the numeric value.
|
||||
//
|
||||
// inline Inline the field, which must be a struct or a map,
|
||||
// causing all of its fields or keys to be processed as if
|
||||
// they were part of the outer struct. For maps, keys must
|
||||
// not conflict with the bson keys of other struct fields.
|
||||
//
|
||||
// Some examples:
|
||||
//
|
||||
// type T struct {
|
||||
// A bool
|
||||
// B int "myb"
|
||||
// C string "myc,omitempty"
|
||||
// D string `bson:",omitempty" json:"jsonkey"`
|
||||
// E int64 ",minsize"
|
||||
// F int64 "myf,omitempty,minsize"
|
||||
// }
|
||||
//
|
||||
func Marshal(in interface{}) (out []byte, err error) {
|
||||
return MarshalBuffer(in, make([]byte, 0, initialBufferSize))
|
||||
}
|
||||
|
||||
// MarshalBuffer behaves the same way as Marshal, except that instead of
|
||||
// allocating a new byte slice it tries to use the received byte slice and
|
||||
// only allocates more memory if necessary to fit the marshaled value.
|
||||
func MarshalBuffer(in interface{}, buf []byte) (out []byte, err error) {
|
||||
defer handleErr(&err)
|
||||
e := &encoder{buf}
|
||||
e.addDoc(reflect.ValueOf(in))
|
||||
return e.out, nil
|
||||
}
|
||||
|
||||
// Unmarshal deserializes data from in into the out value. The out value
|
||||
// must be a map, a pointer to a struct, or a pointer to a bson.D value.
|
||||
// In the case of struct values, only exported fields will be deserialized.
|
||||
// The lowercased field name is used as the key for each exported field,
|
||||
// but this behavior may be changed using the respective field tag.
|
||||
// The tag may also contain flags to tweak the marshalling behavior for
|
||||
// the field. The tag formats accepted are:
|
||||
//
|
||||
// "[<key>][,<flag1>[,<flag2>]]"
|
||||
//
|
||||
// `(...) bson:"[<key>][,<flag1>[,<flag2>]]" (...)`
|
||||
//
|
||||
// The following flags are currently supported during unmarshal (see the
|
||||
// Marshal method for other flags):
|
||||
//
|
||||
// inline Inline the field, which must be a struct or a map.
|
||||
// Inlined structs are handled as if its fields were part
|
||||
// of the outer struct. An inlined map causes keys that do
|
||||
// not match any other struct field to be inserted in the
|
||||
// map rather than being discarded as usual.
|
||||
//
|
||||
// The target field or element types of out may not necessarily match
|
||||
// the BSON values of the provided data. The following conversions are
|
||||
// made automatically:
|
||||
//
|
||||
// - Numeric types are converted if at least the integer part of the
|
||||
// value would be preserved correctly
|
||||
// - Bools are converted to numeric types as 1 or 0
|
||||
// - Numeric types are converted to bools as true if not 0 or false otherwise
|
||||
// - Binary and string BSON data is converted to a string, array or byte slice
|
||||
//
|
||||
// If the value would not fit the type and cannot be converted, it's
|
||||
// silently skipped.
|
||||
//
|
||||
// Pointer values are initialized when necessary.
|
||||
func Unmarshal(in []byte, out interface{}) (err error) {
|
||||
if raw, ok := out.(*Raw); ok {
|
||||
raw.Kind = 3
|
||||
raw.Data = in
|
||||
return nil
|
||||
}
|
||||
defer handleErr(&err)
|
||||
v := reflect.ValueOf(out)
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
fallthrough
|
||||
case reflect.Map:
|
||||
d := newDecoder(in)
|
||||
d.readDocTo(v)
|
||||
if d.i < len(d.in) {
|
||||
return errors.New("document is corrupted")
|
||||
}
|
||||
case reflect.Struct:
|
||||
return errors.New("unmarshal can't deal with struct values. Use a pointer")
|
||||
default:
|
||||
return errors.New("unmarshal needs a map or a pointer to a struct")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmarshal deserializes raw into the out value. If the out value type
|
||||
// is not compatible with raw, a *bson.TypeError is returned.
|
||||
//
|
||||
// See the Unmarshal function documentation for more details on the
|
||||
// unmarshalling process.
|
||||
func (raw Raw) Unmarshal(out interface{}) (err error) {
|
||||
defer handleErr(&err)
|
||||
v := reflect.ValueOf(out)
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
v = v.Elem()
|
||||
fallthrough
|
||||
case reflect.Map:
|
||||
d := newDecoder(raw.Data)
|
||||
good := d.readElemTo(v, raw.Kind)
|
||||
if !good {
|
||||
return &TypeError{v.Type(), raw.Kind}
|
||||
}
|
||||
case reflect.Struct:
|
||||
return errors.New("raw Unmarshal can't deal with struct values. Use a pointer")
|
||||
default:
|
||||
return errors.New("raw Unmarshal needs a map or a valid pointer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TypeError store details for type error occuring
|
||||
// during unmarshaling
|
||||
type TypeError struct {
|
||||
Type reflect.Type
|
||||
Kind byte
|
||||
}
|
||||
|
||||
func (e *TypeError) Error() string {
|
||||
return fmt.Sprintf("BSON kind 0x%02x isn't compatible with type %s", e.Kind, e.Type.String())
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Maintain a mapping of keys to structure field indexes
|
||||
|
||||
type structInfo struct {
|
||||
FieldsMap map[string]fieldInfo
|
||||
FieldsList []fieldInfo
|
||||
InlineMap int
|
||||
Zero reflect.Value
|
||||
}
|
||||
|
||||
type fieldInfo struct {
|
||||
Key string
|
||||
Num int
|
||||
OmitEmpty bool
|
||||
MinSize bool
|
||||
Inline []int
|
||||
}
|
||||
|
||||
var structMap = make(map[reflect.Type]*structInfo)
|
||||
var structMapMutex sync.RWMutex
|
||||
|
||||
type externalPanic string
|
||||
|
||||
func (e externalPanic) String() string {
|
||||
return string(e)
|
||||
}
|
||||
|
||||
func getStructInfo(st reflect.Type) (*structInfo, error) {
|
||||
structMapMutex.RLock()
|
||||
sinfo, found := structMap[st]
|
||||
structMapMutex.RUnlock()
|
||||
if found {
|
||||
return sinfo, nil
|
||||
}
|
||||
n := st.NumField()
|
||||
fieldsMap := make(map[string]fieldInfo)
|
||||
fieldsList := make([]fieldInfo, 0, n)
|
||||
inlineMap := -1
|
||||
for i := 0; i != n; i++ {
|
||||
field := st.Field(i)
|
||||
if field.PkgPath != "" && !field.Anonymous {
|
||||
continue // Private field
|
||||
}
|
||||
|
||||
info := fieldInfo{Num: i}
|
||||
|
||||
tag := field.Tag.Get("bson")
|
||||
|
||||
// Fall-back to JSON struct tag, if feature flag is set.
|
||||
if tag == "" && useJSONTagFallback {
|
||||
tag = field.Tag.Get("json")
|
||||
}
|
||||
|
||||
// If there's no bson/json tag available.
|
||||
if tag == "" {
|
||||
// If there's no tag, and also no tag: value splits (i.e. no colon)
|
||||
// then assume the entire tag is the value
|
||||
if strings.Index(string(field.Tag), ":") < 0 {
|
||||
tag = string(field.Tag)
|
||||
}
|
||||
}
|
||||
|
||||
if tag == "-" {
|
||||
continue
|
||||
}
|
||||
|
||||
inline := false
|
||||
fields := strings.Split(tag, ",")
|
||||
if len(fields) > 1 {
|
||||
for _, flag := range fields[1:] {
|
||||
switch flag {
|
||||
case "omitempty":
|
||||
info.OmitEmpty = true
|
||||
case "minsize":
|
||||
info.MinSize = true
|
||||
case "inline":
|
||||
inline = true
|
||||
default:
|
||||
msg := fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)
|
||||
panic(externalPanic(msg))
|
||||
}
|
||||
}
|
||||
tag = fields[0]
|
||||
}
|
||||
|
||||
if inline {
|
||||
switch field.Type.Kind() {
|
||||
case reflect.Map:
|
||||
if inlineMap >= 0 {
|
||||
return nil, errors.New("Multiple ,inline maps in struct " + st.String())
|
||||
}
|
||||
if field.Type.Key() != reflect.TypeOf("") {
|
||||
return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String())
|
||||
}
|
||||
inlineMap = info.Num
|
||||
case reflect.Ptr:
|
||||
// allow only pointer to struct
|
||||
if kind := field.Type.Elem().Kind(); kind != reflect.Struct {
|
||||
return nil, errors.New("Option ,inline allows a pointer only to a struct, was given pointer to " + kind.String())
|
||||
}
|
||||
|
||||
field.Type = field.Type.Elem()
|
||||
fallthrough
|
||||
case reflect.Struct:
|
||||
sinfo, err := getStructInfo(field.Type)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, finfo := range sinfo.FieldsList {
|
||||
if _, found := fieldsMap[finfo.Key]; found {
|
||||
msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String()
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
if finfo.Inline == nil {
|
||||
finfo.Inline = []int{i, finfo.Num}
|
||||
} else {
|
||||
finfo.Inline = append([]int{i}, finfo.Inline...)
|
||||
}
|
||||
fieldsMap[finfo.Key] = finfo
|
||||
fieldsList = append(fieldsList, finfo)
|
||||
}
|
||||
default:
|
||||
panic("Option ,inline needs a struct value or a pointer to a struct or map field")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if tag != "" {
|
||||
info.Key = tag
|
||||
} else {
|
||||
info.Key = strings.ToLower(field.Name)
|
||||
}
|
||||
|
||||
if _, found = fieldsMap[info.Key]; found {
|
||||
msg := "Duplicated key '" + info.Key + "' in struct " + st.String()
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
|
||||
fieldsList = append(fieldsList, info)
|
||||
fieldsMap[info.Key] = info
|
||||
}
|
||||
sinfo = &structInfo{
|
||||
fieldsMap,
|
||||
fieldsList,
|
||||
inlineMap,
|
||||
reflect.New(st).Elem(),
|
||||
}
|
||||
structMapMutex.Lock()
|
||||
structMap[st] = sinfo
|
||||
structMapMutex.Unlock()
|
||||
return sinfo, nil
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package bson
|
||||
|
||||
// Current state of the JSON tag fallback option.
|
||||
var useJSONTagFallback = false
|
||||
var useRespectNilValues = false
|
||||
|
||||
// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures
|
||||
// without BSON tags on a field will fall-back to using the JSON tag (if present).
|
||||
func SetJSONTagFallback(state bool) {
|
||||
useJSONTagFallback = state
|
||||
}
|
||||
|
||||
// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback
|
||||
// for more information.
|
||||
func JSONTagFallbackState() bool {
|
||||
return useJSONTagFallback
|
||||
}
|
||||
|
||||
// SetRespectNilValues enables or disables serializing nil slices or maps to `null` values.
|
||||
// In other words it enables `encoding/json` compatible behaviour.
|
||||
func SetRespectNilValues(state bool) {
|
||||
useRespectNilValues = state
|
||||
}
|
||||
|
||||
// RespectNilValuesState returns the current status of the JSON nil slices and maps fallback compatibility option.
|
||||
// See SetRespectNilValues for more information.
|
||||
func RespectNilValuesState() bool {
|
||||
return useRespectNilValues
|
||||
}
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
// BSON library for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package bson
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Decimal128 holds decimal128 BSON values.
|
||||
type Decimal128 struct {
|
||||
h, l uint64
|
||||
}
|
||||
|
||||
func (d Decimal128) String() string {
|
||||
var pos int // positive sign
|
||||
var e int // exponent
|
||||
var h, l uint64 // significand high/low
|
||||
|
||||
if d.h>>63&1 == 0 {
|
||||
pos = 1
|
||||
}
|
||||
|
||||
switch d.h >> 58 & (1<<5 - 1) {
|
||||
case 0x1F:
|
||||
return "NaN"
|
||||
case 0x1E:
|
||||
return "-Inf"[pos:]
|
||||
}
|
||||
|
||||
l = d.l
|
||||
if d.h>>61&3 == 3 {
|
||||
// Bits: 1*sign 2*ignored 14*exponent 111*significand.
|
||||
// Implicit 0b100 prefix in significand.
|
||||
e = int(d.h>>47&(1<<14-1)) - 6176
|
||||
//h = 4<<47 | d.h&(1<<47-1)
|
||||
// Spec says all of these values are out of range.
|
||||
h, l = 0, 0
|
||||
} else {
|
||||
// Bits: 1*sign 14*exponent 113*significand
|
||||
e = int(d.h>>49&(1<<14-1)) - 6176
|
||||
h = d.h & (1<<49 - 1)
|
||||
}
|
||||
|
||||
// Would be handled by the logic below, but that's trivial and common.
|
||||
if h == 0 && l == 0 && e == 0 {
|
||||
return "-0"[pos:]
|
||||
}
|
||||
|
||||
var repr [48]byte // Loop 5 times over 9 digits plus dot, negative sign, and leading zero.
|
||||
var last = len(repr)
|
||||
var i = len(repr)
|
||||
var dot = len(repr) + e
|
||||
var rem uint32
|
||||
Loop:
|
||||
for d9 := 0; d9 < 5; d9++ {
|
||||
h, l, rem = divmod(h, l, 1e9)
|
||||
for d1 := 0; d1 < 9; d1++ {
|
||||
// Handle "-0.0", "0.00123400", "-1.00E-6", "1.050E+3", etc.
|
||||
if i < len(repr) && (dot == i || l == 0 && h == 0 && rem > 0 && rem < 10 && (dot < i-6 || e > 0)) {
|
||||
e += len(repr) - i
|
||||
i--
|
||||
repr[i] = '.'
|
||||
last = i - 1
|
||||
dot = len(repr) // Unmark.
|
||||
}
|
||||
c := '0' + byte(rem%10)
|
||||
rem /= 10
|
||||
i--
|
||||
repr[i] = c
|
||||
// Handle "0E+3", "1E+3", etc.
|
||||
if l == 0 && h == 0 && rem == 0 && i == len(repr)-1 && (dot < i-5 || e > 0) {
|
||||
last = i
|
||||
break Loop
|
||||
}
|
||||
if c != '0' {
|
||||
last = i
|
||||
}
|
||||
// Break early. Works without it, but why.
|
||||
if dot > i && l == 0 && h == 0 && rem == 0 {
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
}
|
||||
repr[last-1] = '-'
|
||||
last--
|
||||
|
||||
if e > 0 {
|
||||
return string(repr[last+pos:]) + "E+" + strconv.Itoa(e)
|
||||
}
|
||||
if e < 0 {
|
||||
return string(repr[last+pos:]) + "E" + strconv.Itoa(e)
|
||||
}
|
||||
return string(repr[last+pos:])
|
||||
}
|
||||
|
||||
func divmod(h, l uint64, div uint32) (qh, ql uint64, rem uint32) {
|
||||
div64 := uint64(div)
|
||||
a := h >> 32
|
||||
aq := a / div64
|
||||
ar := a % div64
|
||||
b := ar<<32 + h&(1<<32-1)
|
||||
bq := b / div64
|
||||
br := b % div64
|
||||
c := br<<32 + l>>32
|
||||
cq := c / div64
|
||||
cr := c % div64
|
||||
d := cr<<32 + l&(1<<32-1)
|
||||
dq := d / div64
|
||||
dr := d % div64
|
||||
return (aq<<32 | bq), (cq<<32 | dq), uint32(dr)
|
||||
}
|
||||
|
||||
var dNaN = Decimal128{0x1F << 58, 0}
|
||||
var dPosInf = Decimal128{0x1E << 58, 0}
|
||||
var dNegInf = Decimal128{0x3E << 58, 0}
|
||||
|
||||
func dErr(s string) (Decimal128, error) {
|
||||
return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s)
|
||||
}
|
||||
|
||||
// ParseDecimal128 parse a string and return the corresponding value as
|
||||
// a decimal128
|
||||
func ParseDecimal128(s string) (Decimal128, error) {
|
||||
orig := s
|
||||
if s == "" {
|
||||
return dErr(orig)
|
||||
}
|
||||
neg := s[0] == '-'
|
||||
if neg || s[0] == '+' {
|
||||
s = s[1:]
|
||||
}
|
||||
|
||||
if (len(s) == 3 || len(s) == 8) && (s[0] == 'N' || s[0] == 'n' || s[0] == 'I' || s[0] == 'i') {
|
||||
if s == "NaN" || s == "nan" || strings.EqualFold(s, "nan") {
|
||||
return dNaN, nil
|
||||
}
|
||||
if s == "Inf" || s == "inf" || strings.EqualFold(s, "inf") || strings.EqualFold(s, "infinity") {
|
||||
if neg {
|
||||
return dNegInf, nil
|
||||
}
|
||||
return dPosInf, nil
|
||||
}
|
||||
return dErr(orig)
|
||||
}
|
||||
|
||||
var h, l uint64
|
||||
var e int
|
||||
|
||||
var add, ovr uint32
|
||||
var mul uint32 = 1
|
||||
var dot = -1
|
||||
var digits = 0
|
||||
var i = 0
|
||||
for i < len(s) {
|
||||
c := s[i]
|
||||
if mul == 1e9 {
|
||||
h, l, ovr = muladd(h, l, mul, add)
|
||||
mul, add = 1, 0
|
||||
if ovr > 0 || h&((1<<15-1)<<49) > 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
if c >= '0' && c <= '9' {
|
||||
i++
|
||||
if c > '0' || digits > 0 {
|
||||
digits++
|
||||
}
|
||||
if digits > 34 {
|
||||
if c == '0' {
|
||||
// Exact rounding.
|
||||
e++
|
||||
continue
|
||||
}
|
||||
return dErr(orig)
|
||||
}
|
||||
mul *= 10
|
||||
add *= 10
|
||||
add += uint32(c - '0')
|
||||
continue
|
||||
}
|
||||
if c == '.' {
|
||||
i++
|
||||
if dot >= 0 || i == 1 && len(s) == 1 {
|
||||
return dErr(orig)
|
||||
}
|
||||
if i == len(s) {
|
||||
break
|
||||
}
|
||||
if s[i] < '0' || s[i] > '9' || e > 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
dot = i
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if i == 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
if mul > 1 {
|
||||
h, l, ovr = muladd(h, l, mul, add)
|
||||
if ovr > 0 || h&((1<<15-1)<<49) > 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
if dot >= 0 {
|
||||
e += dot - i
|
||||
}
|
||||
if i+1 < len(s) && (s[i] == 'E' || s[i] == 'e') {
|
||||
i++
|
||||
eneg := s[i] == '-'
|
||||
if eneg || s[i] == '+' {
|
||||
i++
|
||||
if i == len(s) {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for i < len(s) && n < 1e4 {
|
||||
c := s[i]
|
||||
i++
|
||||
if c < '0' || c > '9' {
|
||||
return dErr(orig)
|
||||
}
|
||||
n *= 10
|
||||
n += int(c - '0')
|
||||
}
|
||||
if eneg {
|
||||
n = -n
|
||||
}
|
||||
e += n
|
||||
for e < -6176 {
|
||||
// Subnormal.
|
||||
var div uint32 = 1
|
||||
for div < 1e9 && e < -6176 {
|
||||
div *= 10
|
||||
e++
|
||||
}
|
||||
var rem uint32
|
||||
h, l, rem = divmod(h, l, div)
|
||||
if rem > 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
for e > 6111 {
|
||||
// Clamped.
|
||||
var mul uint32 = 1
|
||||
for mul < 1e9 && e > 6111 {
|
||||
mul *= 10
|
||||
e--
|
||||
}
|
||||
h, l, ovr = muladd(h, l, mul, 0)
|
||||
if ovr > 0 || h&((1<<15-1)<<49) > 0 {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
if e < -6176 || e > 6111 {
|
||||
return dErr(orig)
|
||||
}
|
||||
}
|
||||
|
||||
if i < len(s) {
|
||||
return dErr(orig)
|
||||
}
|
||||
|
||||
h |= uint64(e+6176) & uint64(1<<14-1) << 49
|
||||
if neg {
|
||||
h |= 1 << 63
|
||||
}
|
||||
return Decimal128{h, l}, nil
|
||||
}
|
||||
|
||||
func muladd(h, l uint64, mul uint32, add uint32) (resh, resl uint64, overflow uint32) {
|
||||
mul64 := uint64(mul)
|
||||
a := mul64 * (l & (1<<32 - 1))
|
||||
b := a>>32 + mul64*(l>>32)
|
||||
c := b>>32 + mul64*(h&(1<<32-1))
|
||||
d := c>>32 + mul64*(h>>32)
|
||||
|
||||
a = a&(1<<32-1) + uint64(add)
|
||||
b = b&(1<<32-1) + a>>32
|
||||
c = c&(1<<32-1) + b>>32
|
||||
d = d&(1<<32-1) + c>>32
|
||||
|
||||
return (d<<32 | c&(1<<32-1)), (b<<32 | a&(1<<32-1)), uint32(d >> 32)
|
||||
}
|
||||
-1055
File diff suppressed because it is too large
Load Diff
-645
@@ -1,645 +0,0 @@
|
||||
// BSON library for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
// gobson - BSON library for Go.
|
||||
|
||||
package bson
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Some internal infrastructure.
|
||||
|
||||
var (
|
||||
typeBinary = reflect.TypeOf(Binary{})
|
||||
typeObjectId = reflect.TypeOf(ObjectId(""))
|
||||
typeDBPointer = reflect.TypeOf(DBPointer{"", ObjectId("")})
|
||||
typeSymbol = reflect.TypeOf(Symbol(""))
|
||||
typeMongoTimestamp = reflect.TypeOf(MongoTimestamp(0))
|
||||
typeOrderKey = reflect.TypeOf(MinKey)
|
||||
typeDocElem = reflect.TypeOf(DocElem{})
|
||||
typeRawDocElem = reflect.TypeOf(RawDocElem{})
|
||||
typeRaw = reflect.TypeOf(Raw{})
|
||||
typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{}))
|
||||
typeURL = reflect.TypeOf(url.URL{})
|
||||
typeTime = reflect.TypeOf(time.Time{})
|
||||
typeString = reflect.TypeOf("")
|
||||
typeJSONNumber = reflect.TypeOf(json.Number(""))
|
||||
typeTimeDuration = reflect.TypeOf(time.Duration(0))
|
||||
)
|
||||
|
||||
var (
|
||||
// spec for []uint8 or []byte encoding
|
||||
arrayOps = map[string]bool{
|
||||
"$in": true,
|
||||
"$nin": true,
|
||||
"$all": true,
|
||||
}
|
||||
)
|
||||
|
||||
const itoaCacheSize = 32
|
||||
|
||||
const (
|
||||
getterUnknown = iota
|
||||
getterNone
|
||||
getterTypeVal
|
||||
getterTypePtr
|
||||
getterAddr
|
||||
)
|
||||
|
||||
var itoaCache []string
|
||||
|
||||
var getterStyles map[reflect.Type]int
|
||||
var getterIface reflect.Type
|
||||
var getterMutex sync.RWMutex
|
||||
|
||||
func init() {
|
||||
itoaCache = make([]string, itoaCacheSize)
|
||||
for i := 0; i != itoaCacheSize; i++ {
|
||||
itoaCache[i] = strconv.Itoa(i)
|
||||
}
|
||||
var iface Getter
|
||||
getterIface = reflect.TypeOf(&iface).Elem()
|
||||
getterStyles = make(map[reflect.Type]int)
|
||||
}
|
||||
|
||||
func itoa(i int) string {
|
||||
if i < itoaCacheSize {
|
||||
return itoaCache[i]
|
||||
}
|
||||
return strconv.Itoa(i)
|
||||
}
|
||||
|
||||
func getterStyle(outt reflect.Type) int {
|
||||
getterMutex.RLock()
|
||||
style := getterStyles[outt]
|
||||
getterMutex.RUnlock()
|
||||
if style != getterUnknown {
|
||||
return style
|
||||
}
|
||||
|
||||
getterMutex.Lock()
|
||||
defer getterMutex.Unlock()
|
||||
if outt.Implements(getterIface) {
|
||||
vt := outt
|
||||
for vt.Kind() == reflect.Ptr {
|
||||
vt = vt.Elem()
|
||||
}
|
||||
if vt.Implements(getterIface) {
|
||||
style = getterTypeVal
|
||||
} else {
|
||||
style = getterTypePtr
|
||||
}
|
||||
} else if reflect.PtrTo(outt).Implements(getterIface) {
|
||||
style = getterAddr
|
||||
} else {
|
||||
style = getterNone
|
||||
}
|
||||
getterStyles[outt] = style
|
||||
return style
|
||||
}
|
||||
|
||||
func getGetter(outt reflect.Type, out reflect.Value) Getter {
|
||||
style := getterStyle(outt)
|
||||
if style == getterNone {
|
||||
return nil
|
||||
}
|
||||
if style == getterAddr {
|
||||
if !out.CanAddr() {
|
||||
return nil
|
||||
}
|
||||
return out.Addr().Interface().(Getter)
|
||||
}
|
||||
if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() {
|
||||
return nil
|
||||
}
|
||||
return out.Interface().(Getter)
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Marshaling of the document value itself.
|
||||
|
||||
type encoder struct {
|
||||
out []byte
|
||||
}
|
||||
|
||||
func (e *encoder) addDoc(v reflect.Value) {
|
||||
for {
|
||||
if vi, ok := v.Interface().(Getter); ok {
|
||||
getv, err := vi.GetBSON()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
v = reflect.ValueOf(getv)
|
||||
continue
|
||||
}
|
||||
if v.Kind() == reflect.Ptr {
|
||||
v = v.Elem()
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if v.Type() == typeRaw {
|
||||
raw := v.Interface().(Raw)
|
||||
if raw.Kind != 0x03 && raw.Kind != 0x00 {
|
||||
panic("Attempted to marshal Raw kind " + strconv.Itoa(int(raw.Kind)) + " as a document")
|
||||
}
|
||||
if len(raw.Data) == 0 {
|
||||
panic("Attempted to marshal empty Raw document")
|
||||
}
|
||||
e.addBytes(raw.Data...)
|
||||
return
|
||||
}
|
||||
|
||||
start := e.reserveInt32()
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Map:
|
||||
e.addMap(v)
|
||||
case reflect.Struct:
|
||||
e.addStruct(v)
|
||||
case reflect.Array, reflect.Slice:
|
||||
e.addSlice(v)
|
||||
default:
|
||||
panic("Can't marshal " + v.Type().String() + " as a BSON document")
|
||||
}
|
||||
|
||||
e.addBytes(0)
|
||||
e.setInt32(start, int32(len(e.out)-start))
|
||||
}
|
||||
|
||||
func (e *encoder) addMap(v reflect.Value) {
|
||||
for _, k := range v.MapKeys() {
|
||||
e.addElem(fmt.Sprint(k), v.MapIndex(k), false)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *encoder) addStruct(v reflect.Value) {
|
||||
sinfo, err := getStructInfo(v.Type())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var value reflect.Value
|
||||
if sinfo.InlineMap >= 0 {
|
||||
m := v.Field(sinfo.InlineMap)
|
||||
if m.Len() > 0 {
|
||||
for _, k := range m.MapKeys() {
|
||||
ks := k.String()
|
||||
if _, found := sinfo.FieldsMap[ks]; found {
|
||||
panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", ks))
|
||||
}
|
||||
e.addElem(ks, m.MapIndex(k), false)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, info := range sinfo.FieldsList {
|
||||
if info.Inline == nil {
|
||||
value = v.Field(info.Num)
|
||||
} else {
|
||||
// as pointers to struct are allowed here,
|
||||
// there is no guarantee that pointer won't be nil.
|
||||
//
|
||||
// It is expected allowed behaviour
|
||||
// so info.Inline MAY consist index to a nil pointer
|
||||
// and that is why we safely call v.FieldByIndex and just continue on panic
|
||||
field, errField := safeFieldByIndex(v, info.Inline)
|
||||
if errField != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
value = field
|
||||
}
|
||||
if info.OmitEmpty && isZero(value) {
|
||||
continue
|
||||
}
|
||||
if useRespectNilValues &&
|
||||
(value.Kind() == reflect.Slice || value.Kind() == reflect.Map) &&
|
||||
value.IsNil() {
|
||||
e.addElem(info.Key, reflect.ValueOf(nil), info.MinSize)
|
||||
continue
|
||||
}
|
||||
e.addElem(info.Key, value, info.MinSize)
|
||||
}
|
||||
}
|
||||
|
||||
func safeFieldByIndex(v reflect.Value, index []int) (result reflect.Value, err error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
switch r := recovered.(type) {
|
||||
case string:
|
||||
err = fmt.Errorf("%s", r)
|
||||
case error:
|
||||
err = r
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
result = v.FieldByIndex(index)
|
||||
return
|
||||
}
|
||||
|
||||
func isZero(v reflect.Value) bool {
|
||||
switch v.Kind() {
|
||||
case reflect.String:
|
||||
return len(v.String()) == 0
|
||||
case reflect.Ptr, reflect.Interface:
|
||||
return v.IsNil()
|
||||
case reflect.Slice:
|
||||
return v.Len() == 0
|
||||
case reflect.Map:
|
||||
return v.Len() == 0
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return v.Int() == 0
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
return v.Uint() == 0
|
||||
case reflect.Float32, reflect.Float64:
|
||||
return v.Float() == 0
|
||||
case reflect.Bool:
|
||||
return !v.Bool()
|
||||
case reflect.Struct:
|
||||
vt := v.Type()
|
||||
if vt == typeTime {
|
||||
return v.Interface().(time.Time).IsZero()
|
||||
}
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
if vt.Field(i).PkgPath != "" && !vt.Field(i).Anonymous {
|
||||
continue // Private field
|
||||
}
|
||||
if !isZero(v.Field(i)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *encoder) addSlice(v reflect.Value) {
|
||||
vi := v.Interface()
|
||||
if d, ok := vi.(D); ok {
|
||||
for _, elem := range d {
|
||||
e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
|
||||
}
|
||||
return
|
||||
}
|
||||
if d, ok := vi.(RawD); ok {
|
||||
for _, elem := range d {
|
||||
e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
|
||||
}
|
||||
return
|
||||
}
|
||||
l := v.Len()
|
||||
et := v.Type().Elem()
|
||||
if et == typeDocElem {
|
||||
for i := 0; i < l; i++ {
|
||||
elem := v.Index(i).Interface().(DocElem)
|
||||
e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
|
||||
}
|
||||
return
|
||||
}
|
||||
if et == typeRawDocElem {
|
||||
for i := 0; i < l; i++ {
|
||||
elem := v.Index(i).Interface().(RawDocElem)
|
||||
e.addElem(elem.Name, reflect.ValueOf(elem.Value), false)
|
||||
}
|
||||
return
|
||||
}
|
||||
for i := 0; i < l; i++ {
|
||||
e.addElem(itoa(i), v.Index(i), false)
|
||||
}
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Marshaling of elements in a document.
|
||||
|
||||
func (e *encoder) addElemName(kind byte, name string) {
|
||||
e.addBytes(kind)
|
||||
e.addBytes([]byte(name)...)
|
||||
e.addBytes(0)
|
||||
}
|
||||
|
||||
func (e *encoder) addElem(name string, v reflect.Value, minSize bool) {
|
||||
|
||||
if !v.IsValid() {
|
||||
e.addElemName(0x0A, name)
|
||||
return
|
||||
}
|
||||
|
||||
if getter := getGetter(v.Type(), v); getter != nil {
|
||||
getv, err := getter.GetBSON()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
e.addElem(name, reflect.ValueOf(getv), minSize)
|
||||
return
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
|
||||
case reflect.Interface:
|
||||
e.addElem(name, v.Elem(), minSize)
|
||||
|
||||
case reflect.Ptr:
|
||||
e.addElem(name, v.Elem(), minSize)
|
||||
|
||||
case reflect.String:
|
||||
s := v.String()
|
||||
switch v.Type() {
|
||||
case typeObjectId:
|
||||
if len(s) != 12 {
|
||||
panic("ObjectIDs must be exactly 12 bytes long (got " +
|
||||
strconv.Itoa(len(s)) + ")")
|
||||
}
|
||||
e.addElemName(0x07, name)
|
||||
e.addBytes([]byte(s)...)
|
||||
case typeSymbol:
|
||||
e.addElemName(0x0E, name)
|
||||
e.addStr(s)
|
||||
case typeJSONNumber:
|
||||
n := v.Interface().(json.Number)
|
||||
if i, err := n.Int64(); err == nil {
|
||||
e.addElemName(0x12, name)
|
||||
e.addInt64(i)
|
||||
} else if f, err := n.Float64(); err == nil {
|
||||
e.addElemName(0x01, name)
|
||||
e.addFloat64(f)
|
||||
} else {
|
||||
panic("failed to convert json.Number to a number: " + s)
|
||||
}
|
||||
default:
|
||||
e.addElemName(0x02, name)
|
||||
e.addStr(s)
|
||||
}
|
||||
|
||||
case reflect.Float32, reflect.Float64:
|
||||
e.addElemName(0x01, name)
|
||||
e.addFloat64(v.Float())
|
||||
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
u := v.Uint()
|
||||
if int64(u) < 0 {
|
||||
panic("BSON has no uint64 type, and value is too large to fit correctly in an int64")
|
||||
} else if u <= math.MaxInt32 && (minSize || v.Kind() <= reflect.Uint32) {
|
||||
e.addElemName(0x10, name)
|
||||
e.addInt32(int32(u))
|
||||
} else {
|
||||
e.addElemName(0x12, name)
|
||||
e.addInt64(int64(u))
|
||||
}
|
||||
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
switch v.Type() {
|
||||
case typeMongoTimestamp:
|
||||
e.addElemName(0x11, name)
|
||||
e.addInt64(v.Int())
|
||||
|
||||
case typeOrderKey:
|
||||
if v.Int() == int64(MaxKey) {
|
||||
e.addElemName(0x7F, name)
|
||||
} else {
|
||||
e.addElemName(0xFF, name)
|
||||
}
|
||||
case typeTimeDuration:
|
||||
// Stored as int64
|
||||
e.addElemName(0x12, name)
|
||||
|
||||
e.addInt64(int64(v.Int() / 1e6))
|
||||
default:
|
||||
i := v.Int()
|
||||
if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 {
|
||||
// It fits into an int32, encode as such.
|
||||
e.addElemName(0x10, name)
|
||||
e.addInt32(int32(i))
|
||||
} else {
|
||||
e.addElemName(0x12, name)
|
||||
e.addInt64(i)
|
||||
}
|
||||
}
|
||||
|
||||
case reflect.Bool:
|
||||
e.addElemName(0x08, name)
|
||||
if v.Bool() {
|
||||
e.addBytes(1)
|
||||
} else {
|
||||
e.addBytes(0)
|
||||
}
|
||||
|
||||
case reflect.Map:
|
||||
e.addElemName(0x03, name)
|
||||
e.addDoc(v)
|
||||
|
||||
case reflect.Slice:
|
||||
vt := v.Type()
|
||||
et := vt.Elem()
|
||||
if et.Kind() == reflect.Uint8 {
|
||||
if arrayOps[name] {
|
||||
e.addElemName(0x04, name)
|
||||
e.addDoc(v)
|
||||
} else {
|
||||
e.addElemName(0x05, name)
|
||||
e.addBinary(0x00, v.Bytes())
|
||||
}
|
||||
} else if et == typeDocElem || et == typeRawDocElem {
|
||||
e.addElemName(0x03, name)
|
||||
e.addDoc(v)
|
||||
} else {
|
||||
e.addElemName(0x04, name)
|
||||
e.addDoc(v)
|
||||
}
|
||||
|
||||
case reflect.Array:
|
||||
et := v.Type().Elem()
|
||||
if et.Kind() == reflect.Uint8 {
|
||||
if arrayOps[name] {
|
||||
e.addElemName(0x04, name)
|
||||
e.addDoc(v)
|
||||
} else {
|
||||
e.addElemName(0x05, name)
|
||||
if v.CanAddr() {
|
||||
e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte))
|
||||
} else {
|
||||
n := v.Len()
|
||||
e.addInt32(int32(n))
|
||||
e.addBytes(0x00)
|
||||
for i := 0; i < n; i++ {
|
||||
el := v.Index(i)
|
||||
e.addBytes(byte(el.Uint()))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
e.addElemName(0x04, name)
|
||||
e.addDoc(v)
|
||||
}
|
||||
|
||||
case reflect.Struct:
|
||||
switch s := v.Interface().(type) {
|
||||
|
||||
case Raw:
|
||||
kind := s.Kind
|
||||
if kind == 0x00 {
|
||||
kind = 0x03
|
||||
}
|
||||
if len(s.Data) == 0 && kind != 0x06 && kind != 0x0A && kind != 0xFF && kind != 0x7F {
|
||||
panic("Attempted to marshal empty Raw document")
|
||||
}
|
||||
e.addElemName(kind, name)
|
||||
e.addBytes(s.Data...)
|
||||
|
||||
case Binary:
|
||||
e.addElemName(0x05, name)
|
||||
e.addBinary(s.Kind, s.Data)
|
||||
|
||||
case Decimal128:
|
||||
e.addElemName(0x13, name)
|
||||
e.addInt64(int64(s.l))
|
||||
e.addInt64(int64(s.h))
|
||||
|
||||
case DBPointer:
|
||||
e.addElemName(0x0C, name)
|
||||
e.addStr(s.Namespace)
|
||||
if len(s.Id) != 12 {
|
||||
panic("ObjectIDs must be exactly 12 bytes long (got " +
|
||||
strconv.Itoa(len(s.Id)) + ")")
|
||||
}
|
||||
e.addBytes([]byte(s.Id)...)
|
||||
|
||||
case RegEx:
|
||||
e.addElemName(0x0B, name)
|
||||
e.addCStr(s.Pattern)
|
||||
options := runes(s.Options)
|
||||
sort.Sort(options)
|
||||
e.addCStr(string(options))
|
||||
|
||||
case JavaScript:
|
||||
if s.Scope == nil {
|
||||
e.addElemName(0x0D, name)
|
||||
e.addStr(s.Code)
|
||||
} else {
|
||||
e.addElemName(0x0F, name)
|
||||
start := e.reserveInt32()
|
||||
e.addStr(s.Code)
|
||||
e.addDoc(reflect.ValueOf(s.Scope))
|
||||
e.setInt32(start, int32(len(e.out)-start))
|
||||
}
|
||||
|
||||
case time.Time:
|
||||
// MongoDB handles timestamps as milliseconds.
|
||||
e.addElemName(0x09, name)
|
||||
e.addInt64(s.Unix()*1000 + int64(s.Nanosecond()/1e6))
|
||||
|
||||
case url.URL:
|
||||
e.addElemName(0x02, name)
|
||||
e.addStr(s.String())
|
||||
|
||||
case undefined:
|
||||
e.addElemName(0x06, name)
|
||||
|
||||
default:
|
||||
e.addElemName(0x03, name)
|
||||
e.addDoc(v)
|
||||
}
|
||||
|
||||
default:
|
||||
panic("Can't marshal " + v.Type().String() + " in a BSON document")
|
||||
}
|
||||
}
|
||||
|
||||
// -------------
|
||||
// Helper method for sorting regex options
|
||||
type runes []rune
|
||||
|
||||
func (a runes) Len() int { return len(a) }
|
||||
func (a runes) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a runes) Less(i, j int) bool { return a[i] < a[j] }
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Marshaling of base types.
|
||||
|
||||
func (e *encoder) addBinary(subtype byte, v []byte) {
|
||||
if subtype == 0x02 {
|
||||
// Wonder how that brilliant idea came to life. Obsolete, luckily.
|
||||
e.addInt32(int32(len(v) + 4))
|
||||
e.addBytes(subtype)
|
||||
e.addInt32(int32(len(v)))
|
||||
} else {
|
||||
e.addInt32(int32(len(v)))
|
||||
e.addBytes(subtype)
|
||||
}
|
||||
e.addBytes(v...)
|
||||
}
|
||||
|
||||
func (e *encoder) addStr(v string) {
|
||||
e.addInt32(int32(len(v) + 1))
|
||||
e.addCStr(v)
|
||||
}
|
||||
|
||||
func (e *encoder) addCStr(v string) {
|
||||
e.addBytes([]byte(v)...)
|
||||
e.addBytes(0)
|
||||
}
|
||||
|
||||
func (e *encoder) reserveInt32() (pos int) {
|
||||
pos = len(e.out)
|
||||
e.addBytes(0, 0, 0, 0)
|
||||
return pos
|
||||
}
|
||||
|
||||
func (e *encoder) setInt32(pos int, v int32) {
|
||||
e.out[pos+0] = byte(v)
|
||||
e.out[pos+1] = byte(v >> 8)
|
||||
e.out[pos+2] = byte(v >> 16)
|
||||
e.out[pos+3] = byte(v >> 24)
|
||||
}
|
||||
|
||||
func (e *encoder) addInt32(v int32) {
|
||||
u := uint32(v)
|
||||
e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24))
|
||||
}
|
||||
|
||||
func (e *encoder) addInt64(v int64) {
|
||||
u := uint64(v)
|
||||
e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24),
|
||||
byte(u>>32), byte(u>>40), byte(u>>48), byte(u>>56))
|
||||
}
|
||||
|
||||
func (e *encoder) addFloat64(v float64) {
|
||||
e.addInt64(int64(math.Float64bits(v)))
|
||||
}
|
||||
|
||||
func (e *encoder) addBytes(v ...byte) {
|
||||
e.out = append(e.out, v...)
|
||||
}
|
||||
-384
@@ -1,384 +0,0 @@
|
||||
package bson
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/internal/json"
|
||||
)
|
||||
|
||||
// UnmarshalJSON unmarshals a JSON value that may hold non-standard
|
||||
// syntax as defined in BSON's extended JSON specification.
|
||||
func UnmarshalJSON(data []byte, value interface{}) error {
|
||||
d := json.NewDecoder(bytes.NewBuffer(data))
|
||||
d.Extend(&jsonExt)
|
||||
return d.Decode(value)
|
||||
}
|
||||
|
||||
// MarshalJSON marshals a JSON value that may hold non-standard
|
||||
// syntax as defined in BSON's extended JSON specification.
|
||||
func MarshalJSON(value interface{}) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
e := json.NewEncoder(&buf)
|
||||
e.Extend(&jsonExt)
|
||||
err := e.Encode(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// jdec is used internally by the JSON decoding functions
|
||||
// so they may unmarshal functions without getting into endless
|
||||
// recursion due to keyed objects.
|
||||
func jdec(data []byte, value interface{}) error {
|
||||
d := json.NewDecoder(bytes.NewBuffer(data))
|
||||
d.Extend(&funcExt)
|
||||
return d.Decode(value)
|
||||
}
|
||||
|
||||
var jsonExt json.Extension
|
||||
var funcExt json.Extension
|
||||
|
||||
// TODO
|
||||
// - Shell regular expressions ("/regexp/opts")
|
||||
|
||||
func init() {
|
||||
jsonExt.DecodeUnquotedKeys(true)
|
||||
jsonExt.DecodeTrailingCommas(true)
|
||||
|
||||
funcExt.DecodeFunc("BinData", "$binaryFunc", "$type", "$binary")
|
||||
jsonExt.DecodeKeyed("$binary", jdecBinary)
|
||||
jsonExt.DecodeKeyed("$binaryFunc", jdecBinary)
|
||||
jsonExt.EncodeType([]byte(nil), jencBinarySlice)
|
||||
jsonExt.EncodeType(Binary{}, jencBinaryType)
|
||||
|
||||
funcExt.DecodeFunc("ISODate", "$dateFunc", "S")
|
||||
funcExt.DecodeFunc("new Date", "$dateFunc", "S")
|
||||
jsonExt.DecodeKeyed("$date", jdecDate)
|
||||
jsonExt.DecodeKeyed("$dateFunc", jdecDate)
|
||||
jsonExt.EncodeType(time.Time{}, jencDate)
|
||||
|
||||
funcExt.DecodeFunc("Timestamp", "$timestamp", "t", "i")
|
||||
jsonExt.DecodeKeyed("$timestamp", jdecTimestamp)
|
||||
jsonExt.EncodeType(MongoTimestamp(0), jencTimestamp)
|
||||
|
||||
funcExt.DecodeConst("undefined", Undefined)
|
||||
|
||||
jsonExt.DecodeKeyed("$regex", jdecRegEx)
|
||||
jsonExt.EncodeType(RegEx{}, jencRegEx)
|
||||
|
||||
funcExt.DecodeFunc("ObjectId", "$oidFunc", "Id")
|
||||
jsonExt.DecodeKeyed("$oid", jdecObjectId)
|
||||
jsonExt.DecodeKeyed("$oidFunc", jdecObjectId)
|
||||
jsonExt.EncodeType(ObjectId(""), jencObjectId)
|
||||
|
||||
funcExt.DecodeFunc("DBRef", "$dbrefFunc", "$ref", "$id")
|
||||
jsonExt.DecodeKeyed("$dbrefFunc", jdecDBRef)
|
||||
|
||||
funcExt.DecodeFunc("NumberLong", "$numberLongFunc", "N")
|
||||
jsonExt.DecodeKeyed("$numberLong", jdecNumberLong)
|
||||
jsonExt.DecodeKeyed("$numberLongFunc", jdecNumberLong)
|
||||
jsonExt.EncodeType(int64(0), jencNumberLong)
|
||||
jsonExt.EncodeType(int(0), jencInt)
|
||||
|
||||
funcExt.DecodeConst("MinKey", MinKey)
|
||||
funcExt.DecodeConst("MaxKey", MaxKey)
|
||||
jsonExt.DecodeKeyed("$minKey", jdecMinKey)
|
||||
jsonExt.DecodeKeyed("$maxKey", jdecMaxKey)
|
||||
jsonExt.EncodeType(orderKey(0), jencMinMaxKey)
|
||||
|
||||
jsonExt.DecodeKeyed("$undefined", jdecUndefined)
|
||||
jsonExt.EncodeType(Undefined, jencUndefined)
|
||||
|
||||
jsonExt.Extend(&funcExt)
|
||||
}
|
||||
|
||||
func fbytes(format string, args ...interface{}) []byte {
|
||||
var buf bytes.Buffer
|
||||
fmt.Fprintf(&buf, format, args...)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func jdecBinary(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
Binary []byte `json:"$binary"`
|
||||
Type string `json:"$type"`
|
||||
Func struct {
|
||||
Binary []byte `json:"$binary"`
|
||||
Type int64 `json:"$type"`
|
||||
} `json:"$binaryFunc"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var binData []byte
|
||||
var binKind int64
|
||||
if v.Type == "" && v.Binary == nil {
|
||||
binData = v.Func.Binary
|
||||
binKind = v.Func.Type
|
||||
} else if v.Type == "" {
|
||||
return v.Binary, nil
|
||||
} else {
|
||||
binData = v.Binary
|
||||
binKind, err = strconv.ParseInt(v.Type, 0, 64)
|
||||
if err != nil {
|
||||
binKind = -1
|
||||
}
|
||||
}
|
||||
|
||||
if binKind == 0 {
|
||||
return binData, nil
|
||||
}
|
||||
if binKind < 0 || binKind > 255 {
|
||||
return nil, fmt.Errorf("invalid type in binary object: %s", data)
|
||||
}
|
||||
|
||||
return Binary{Kind: byte(binKind), Data: binData}, nil
|
||||
}
|
||||
|
||||
func jencBinarySlice(v interface{}) ([]byte, error) {
|
||||
in := v.([]byte)
|
||||
out := make([]byte, base64.StdEncoding.EncodedLen(len(in)))
|
||||
base64.StdEncoding.Encode(out, in)
|
||||
return fbytes(`{"$binary":"%s","$type":"0x0"}`, out), nil
|
||||
}
|
||||
|
||||
func jencBinaryType(v interface{}) ([]byte, error) {
|
||||
in := v.(Binary)
|
||||
out := make([]byte, base64.StdEncoding.EncodedLen(len(in.Data)))
|
||||
base64.StdEncoding.Encode(out, in.Data)
|
||||
return fbytes(`{"$binary":"%s","$type":"0x%x"}`, out, in.Kind), nil
|
||||
}
|
||||
|
||||
const jdateFormat = "2006-01-02T15:04:05.999Z07:00"
|
||||
|
||||
func jdecDate(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
S string `json:"$date"`
|
||||
Func struct {
|
||||
S string
|
||||
} `json:"$dateFunc"`
|
||||
}
|
||||
_ = jdec(data, &v)
|
||||
if v.S == "" {
|
||||
v.S = v.Func.S
|
||||
}
|
||||
if v.S != "" {
|
||||
var errs []string
|
||||
for _, format := range []string{jdateFormat, "2006-01-02"} {
|
||||
t, err := time.Parse(format, v.S)
|
||||
if err == nil {
|
||||
return t, nil
|
||||
}
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
return nil, fmt.Errorf("cannot parse date: %q [%s]", v.S, strings.Join(errs, ", "))
|
||||
}
|
||||
|
||||
var vn struct {
|
||||
Date struct {
|
||||
N int64 `json:"$numberLong,string"`
|
||||
} `json:"$date"`
|
||||
Func struct {
|
||||
S int64
|
||||
} `json:"$dateFunc"`
|
||||
}
|
||||
err := jdec(data, &vn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse date: %q", data)
|
||||
}
|
||||
n := vn.Date.N
|
||||
if n == 0 {
|
||||
n = vn.Func.S
|
||||
}
|
||||
return time.Unix(n/1000, n%1000*1e6).UTC(), nil
|
||||
}
|
||||
|
||||
func jencDate(v interface{}) ([]byte, error) {
|
||||
t := v.(time.Time)
|
||||
return fbytes(`{"$date":%q}`, t.Format(jdateFormat)), nil
|
||||
}
|
||||
|
||||
func jdecTimestamp(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
Func struct {
|
||||
T int32 `json:"t"`
|
||||
I int32 `json:"i"`
|
||||
} `json:"$timestamp"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return MongoTimestamp(uint64(v.Func.T)<<32 | uint64(uint32(v.Func.I))), nil
|
||||
}
|
||||
|
||||
func jencTimestamp(v interface{}) ([]byte, error) {
|
||||
ts := uint64(v.(MongoTimestamp))
|
||||
return fbytes(`{"$timestamp":{"t":%d,"i":%d}}`, ts>>32, uint32(ts)), nil
|
||||
}
|
||||
|
||||
func jdecRegEx(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
Regex string `json:"$regex"`
|
||||
Options string `json:"$options"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return RegEx{v.Regex, v.Options}, nil
|
||||
}
|
||||
|
||||
func jencRegEx(v interface{}) ([]byte, error) {
|
||||
re := v.(RegEx)
|
||||
type regex struct {
|
||||
Regex string `json:"$regex"`
|
||||
Options string `json:"$options"`
|
||||
}
|
||||
return json.Marshal(regex{re.Pattern, re.Options})
|
||||
}
|
||||
|
||||
func jdecObjectId(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
Id string `json:"$oid"`
|
||||
Func struct {
|
||||
Id string
|
||||
} `json:"$oidFunc"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.Id == "" {
|
||||
v.Id = v.Func.Id
|
||||
}
|
||||
return ObjectIdHex(v.Id), nil
|
||||
}
|
||||
|
||||
func jencObjectId(v interface{}) ([]byte, error) {
|
||||
return fbytes(`{"$oid":"%s"}`, v.(ObjectId).Hex()), nil
|
||||
}
|
||||
|
||||
func jdecDBRef(data []byte) (interface{}, error) {
|
||||
// TODO Support unmarshaling $ref and $id into the input value.
|
||||
var v struct {
|
||||
Obj map[string]interface{} `json:"$dbrefFunc"`
|
||||
}
|
||||
// TODO Fix this. Must not be required.
|
||||
v.Obj = make(map[string]interface{})
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v.Obj, nil
|
||||
}
|
||||
|
||||
func jdecNumberLong(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
N int64 `json:"$numberLong,string"`
|
||||
Func struct {
|
||||
N int64 `json:",string"`
|
||||
} `json:"$numberLongFunc"`
|
||||
}
|
||||
var vn struct {
|
||||
N int64 `json:"$numberLong"`
|
||||
Func struct {
|
||||
N int64
|
||||
} `json:"$numberLongFunc"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
err = jdec(data, &vn)
|
||||
v.N = vn.N
|
||||
v.Func.N = vn.Func.N
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.N != 0 {
|
||||
return v.N, nil
|
||||
}
|
||||
return v.Func.N, nil
|
||||
}
|
||||
|
||||
func jencNumberLong(v interface{}) ([]byte, error) {
|
||||
n := v.(int64)
|
||||
f := `{"$numberLong":"%d"}`
|
||||
if n <= 1<<53 {
|
||||
f = `{"$numberLong":%d}`
|
||||
}
|
||||
return fbytes(f, n), nil
|
||||
}
|
||||
|
||||
func jencInt(v interface{}) ([]byte, error) {
|
||||
n := v.(int)
|
||||
f := `{"$numberLong":"%d"}`
|
||||
if int64(n) <= 1<<53 {
|
||||
f = `%d`
|
||||
}
|
||||
return fbytes(f, n), nil
|
||||
}
|
||||
|
||||
func jdecMinKey(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
N int64 `json:"$minKey"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.N != 1 {
|
||||
return nil, fmt.Errorf("invalid $minKey object: %s", data)
|
||||
}
|
||||
return MinKey, nil
|
||||
}
|
||||
|
||||
func jdecMaxKey(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
N int64 `json:"$maxKey"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.N != 1 {
|
||||
return nil, fmt.Errorf("invalid $maxKey object: %s", data)
|
||||
}
|
||||
return MaxKey, nil
|
||||
}
|
||||
|
||||
func jencMinMaxKey(v interface{}) ([]byte, error) {
|
||||
switch v.(orderKey) {
|
||||
case MinKey:
|
||||
return []byte(`{"$minKey":1}`), nil
|
||||
case MaxKey:
|
||||
return []byte(`{"$maxKey":1}`), nil
|
||||
}
|
||||
panic(fmt.Sprintf("invalid $minKey/$maxKey value: %d", v))
|
||||
}
|
||||
|
||||
func jdecUndefined(data []byte) (interface{}, error) {
|
||||
var v struct {
|
||||
B bool `json:"$undefined"`
|
||||
}
|
||||
err := jdec(data, &v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !v.B {
|
||||
return nil, fmt.Errorf("invalid $undefined object: %s", data)
|
||||
}
|
||||
return Undefined, nil
|
||||
}
|
||||
|
||||
func jencUndefined(v interface{}) ([]byte, error) {
|
||||
return []byte(`{"$undefined":true}`), nil
|
||||
}
|
||||
-90
@@ -1,90 +0,0 @@
|
||||
package bson
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
// MinDocumentSize is the size of the smallest possible valid BSON document:
|
||||
// an int32 size header + 0x00 (end of document).
|
||||
MinDocumentSize = 5
|
||||
|
||||
// MaxDocumentSize is the largest possible size for a BSON document allowed by MongoDB,
|
||||
// that is, 16 MiB (see https://docs.mongodb.com/manual/reference/limits/).
|
||||
MaxDocumentSize = 16777216
|
||||
)
|
||||
|
||||
// ErrInvalidDocumentSize is an error returned when a BSON document's header
|
||||
// contains a size smaller than MinDocumentSize or greater than MaxDocumentSize.
|
||||
type ErrInvalidDocumentSize struct {
|
||||
DocumentSize int32
|
||||
}
|
||||
|
||||
func (e ErrInvalidDocumentSize) Error() string {
|
||||
return fmt.Sprintf("invalid document size %d", e.DocumentSize)
|
||||
}
|
||||
|
||||
// A Decoder reads and decodes BSON values from an input stream.
|
||||
type Decoder struct {
|
||||
source io.Reader
|
||||
}
|
||||
|
||||
// NewDecoder returns a new Decoder that reads from source.
|
||||
// It does not add any extra buffering, and may not read data from source beyond the BSON values requested.
|
||||
func NewDecoder(source io.Reader) *Decoder {
|
||||
return &Decoder{source: source}
|
||||
}
|
||||
|
||||
// Decode reads the next BSON-encoded value from its input and stores it in the value pointed to by v.
|
||||
// See the documentation for Unmarshal for details about the conversion of BSON into a Go value.
|
||||
func (dec *Decoder) Decode(v interface{}) (err error) {
|
||||
// BSON documents start with their size as a *signed* int32.
|
||||
var docSize int32
|
||||
if err = binary.Read(dec.source, binary.LittleEndian, &docSize); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if docSize < MinDocumentSize || docSize > MaxDocumentSize {
|
||||
return ErrInvalidDocumentSize{DocumentSize: docSize}
|
||||
}
|
||||
|
||||
docBuffer := bytes.NewBuffer(make([]byte, 0, docSize))
|
||||
if err = binary.Write(docBuffer, binary.LittleEndian, docSize); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// docSize is the *full* document's size (including the 4-byte size header,
|
||||
// which has already been read).
|
||||
if _, err = io.CopyN(docBuffer, dec.source, int64(docSize-4)); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Let Unmarshal handle the rest.
|
||||
defer handleErr(&err)
|
||||
return Unmarshal(docBuffer.Bytes(), v)
|
||||
}
|
||||
|
||||
// An Encoder encodes and writes BSON values to an output stream.
|
||||
type Encoder struct {
|
||||
target io.Writer
|
||||
}
|
||||
|
||||
// NewEncoder returns a new Encoder that writes to target.
|
||||
func NewEncoder(target io.Writer) *Encoder {
|
||||
return &Encoder{target: target}
|
||||
}
|
||||
|
||||
// Encode encodes v to BSON, and if successful writes it to the Encoder's output stream.
|
||||
// See the documentation for Marshal for details about the conversion of Go values to BSON.
|
||||
func (enc *Encoder) Encode(v interface{}) error {
|
||||
data, err := Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = enc.target.Write(data)
|
||||
return err
|
||||
}
|
||||
-366
@@ -1,366 +0,0 @@
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
// Bulk represents an operation that can be prepared with several
|
||||
// orthogonal changes before being delivered to the server.
|
||||
//
|
||||
// MongoDB servers older than version 2.6 do not have proper support for bulk
|
||||
// operations, so the driver attempts to map its API as much as possible into
|
||||
// the functionality that works. In particular, in those releases updates and
|
||||
// removals are sent individually, and inserts are sent in bulk but have
|
||||
// suboptimal error reporting compared to more recent versions of the server.
|
||||
// See the documentation of BulkErrorCase for details on that.
|
||||
//
|
||||
// Relevant documentation:
|
||||
//
|
||||
// http://blog.mongodb.org/post/84922794768/mongodbs-new-bulk-api
|
||||
//
|
||||
type Bulk struct {
|
||||
c *Collection
|
||||
opcount int
|
||||
actions []bulkAction
|
||||
ordered bool
|
||||
}
|
||||
|
||||
type bulkOp int
|
||||
|
||||
const (
|
||||
bulkInsert bulkOp = iota + 1
|
||||
bulkUpdate
|
||||
bulkUpdateAll
|
||||
bulkRemove
|
||||
)
|
||||
|
||||
type bulkAction struct {
|
||||
op bulkOp
|
||||
docs []interface{}
|
||||
idxs []int
|
||||
}
|
||||
|
||||
type bulkUpdateOp []interface{}
|
||||
type bulkDeleteOp []interface{}
|
||||
|
||||
// BulkResult holds the results for a bulk operation.
|
||||
type BulkResult struct {
|
||||
Matched int
|
||||
Modified int // Available only for MongoDB 2.6+
|
||||
|
||||
// Be conservative while we understand exactly how to report these
|
||||
// results in a useful and convenient way, and also how to emulate
|
||||
// them with prior servers.
|
||||
private bool
|
||||
}
|
||||
|
||||
// BulkError holds an error returned from running a Bulk operation.
|
||||
// Individual errors may be obtained and inspected via the Cases method.
|
||||
type BulkError struct {
|
||||
ecases []BulkErrorCase
|
||||
}
|
||||
|
||||
func (e *BulkError) Error() string {
|
||||
if len(e.ecases) == 0 {
|
||||
return "invalid BulkError instance: no errors"
|
||||
}
|
||||
if len(e.ecases) == 1 {
|
||||
return e.ecases[0].Err.Error()
|
||||
}
|
||||
msgs := make([]string, 0, len(e.ecases))
|
||||
seen := make(map[string]bool)
|
||||
for _, ecase := range e.ecases {
|
||||
msg := ecase.Err.Error()
|
||||
if !seen[msg] {
|
||||
seen[msg] = true
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
}
|
||||
if len(msgs) == 1 {
|
||||
return msgs[0]
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("multiple errors in bulk operation:\n")
|
||||
for _, msg := range msgs {
|
||||
buf.WriteString(" - ")
|
||||
buf.WriteString(msg)
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
type bulkErrorCases []BulkErrorCase
|
||||
|
||||
func (slice bulkErrorCases) Len() int { return len(slice) }
|
||||
func (slice bulkErrorCases) Less(i, j int) bool { return slice[i].Index < slice[j].Index }
|
||||
func (slice bulkErrorCases) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] }
|
||||
|
||||
// BulkErrorCase holds an individual error found while attempting a single change
|
||||
// within a bulk operation, and the position in which it was enqueued.
|
||||
//
|
||||
// MongoDB servers older than version 2.6 do not have proper support for bulk
|
||||
// operations, so the driver attempts to map its API as much as possible into
|
||||
// the functionality that works. In particular, only the last error is reported
|
||||
// for bulk inserts and without any positional information, so the Index
|
||||
// field is set to -1 in these cases.
|
||||
type BulkErrorCase struct {
|
||||
Index int // Position of operation that failed, or -1 if unknown.
|
||||
Err error
|
||||
}
|
||||
|
||||
// Cases returns all individual errors found while attempting the requested changes.
|
||||
//
|
||||
// See the documentation of BulkErrorCase for limitations in older MongoDB releases.
|
||||
func (e *BulkError) Cases() []BulkErrorCase {
|
||||
return e.ecases
|
||||
}
|
||||
|
||||
var actionPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &bulkAction{
|
||||
docs: make([]interface{}, 0),
|
||||
idxs: make([]int, 0),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// Bulk returns a value to prepare the execution of a bulk operation.
|
||||
func (c *Collection) Bulk() *Bulk {
|
||||
return &Bulk{c: c, ordered: true}
|
||||
}
|
||||
|
||||
// Unordered puts the bulk operation in unordered mode.
|
||||
//
|
||||
// In unordered mode the indvidual operations may be sent
|
||||
// out of order, which means latter operations may proceed
|
||||
// even if prior ones have failed.
|
||||
func (b *Bulk) Unordered() {
|
||||
b.ordered = false
|
||||
}
|
||||
|
||||
func (b *Bulk) action(op bulkOp, opcount int) *bulkAction {
|
||||
var action *bulkAction
|
||||
if len(b.actions) > 0 && b.actions[len(b.actions)-1].op == op {
|
||||
action = &b.actions[len(b.actions)-1]
|
||||
} else if !b.ordered {
|
||||
for i := range b.actions {
|
||||
if b.actions[i].op == op {
|
||||
action = &b.actions[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if action == nil {
|
||||
a := actionPool.Get().(*bulkAction)
|
||||
a.op = op
|
||||
b.actions = append(b.actions, *a)
|
||||
action = &b.actions[len(b.actions)-1]
|
||||
}
|
||||
for i := 0; i < opcount; i++ {
|
||||
action.idxs = append(action.idxs, b.opcount)
|
||||
b.opcount++
|
||||
}
|
||||
return action
|
||||
}
|
||||
|
||||
// Insert queues up the provided documents for insertion.
|
||||
func (b *Bulk) Insert(docs ...interface{}) {
|
||||
action := b.action(bulkInsert, len(docs))
|
||||
action.docs = append(action.docs, docs...)
|
||||
}
|
||||
|
||||
// Remove queues up the provided selectors for removing matching documents.
|
||||
// Each selector will remove only a single matching document.
|
||||
func (b *Bulk) Remove(selectors ...interface{}) {
|
||||
action := b.action(bulkRemove, len(selectors))
|
||||
for _, selector := range selectors {
|
||||
if selector == nil {
|
||||
selector = bson.D{}
|
||||
}
|
||||
action.docs = append(action.docs, &deleteOp{
|
||||
Collection: b.c.FullName,
|
||||
Selector: selector,
|
||||
Flags: 1,
|
||||
Limit: 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveAll queues up the provided selectors for removing all matching documents.
|
||||
// Each selector will remove all matching documents.
|
||||
func (b *Bulk) RemoveAll(selectors ...interface{}) {
|
||||
action := b.action(bulkRemove, len(selectors))
|
||||
for _, selector := range selectors {
|
||||
if selector == nil {
|
||||
selector = bson.D{}
|
||||
}
|
||||
action.docs = append(action.docs, &deleteOp{
|
||||
Collection: b.c.FullName,
|
||||
Selector: selector,
|
||||
Flags: 0,
|
||||
Limit: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Update queues up the provided pairs of updating instructions.
|
||||
// The first element of each pair selects which documents must be
|
||||
// updated, and the second element defines how to update it.
|
||||
// Each pair matches exactly one document for updating at most.
|
||||
func (b *Bulk) Update(pairs ...interface{}) {
|
||||
if len(pairs)%2 != 0 {
|
||||
panic("Bulk.Update requires an even number of parameters")
|
||||
}
|
||||
action := b.action(bulkUpdate, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
selector := pairs[i]
|
||||
if selector == nil {
|
||||
selector = bson.D{}
|
||||
}
|
||||
action.docs = append(action.docs, &updateOp{
|
||||
Collection: b.c.FullName,
|
||||
Selector: selector,
|
||||
Update: pairs[i+1],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateAll queues up the provided pairs of updating instructions.
|
||||
// The first element of each pair selects which documents must be
|
||||
// updated, and the second element defines how to update it.
|
||||
// Each pair updates all documents matching the selector.
|
||||
func (b *Bulk) UpdateAll(pairs ...interface{}) {
|
||||
if len(pairs)%2 != 0 {
|
||||
panic("Bulk.UpdateAll requires an even number of parameters")
|
||||
}
|
||||
action := b.action(bulkUpdate, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
selector := pairs[i]
|
||||
if selector == nil {
|
||||
selector = bson.D{}
|
||||
}
|
||||
action.docs = append(action.docs, &updateOp{
|
||||
Collection: b.c.FullName,
|
||||
Selector: selector,
|
||||
Update: pairs[i+1],
|
||||
Flags: 2,
|
||||
Multi: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Upsert queues up the provided pairs of upserting instructions.
|
||||
// The first element of each pair selects which documents must be
|
||||
// updated, and the second element defines how to update it.
|
||||
// Each pair matches exactly one document for updating at most.
|
||||
func (b *Bulk) Upsert(pairs ...interface{}) {
|
||||
if len(pairs)%2 != 0 {
|
||||
panic("Bulk.Update requires an even number of parameters")
|
||||
}
|
||||
action := b.action(bulkUpdate, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
selector := pairs[i]
|
||||
if selector == nil {
|
||||
selector = bson.D{}
|
||||
}
|
||||
action.docs = append(action.docs, &updateOp{
|
||||
Collection: b.c.FullName,
|
||||
Selector: selector,
|
||||
Update: pairs[i+1],
|
||||
Flags: 1,
|
||||
Upsert: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Run runs all the operations queued up.
|
||||
//
|
||||
// If an error is reported on an unordered bulk operation, the error value may
|
||||
// be an aggregation of all issues observed. As an exception to that, Insert
|
||||
// operations running on MongoDB versions prior to 2.6 will report the last
|
||||
// error only due to a limitation in the wire protocol.
|
||||
func (b *Bulk) Run() (*BulkResult, error) {
|
||||
var result BulkResult
|
||||
var berr BulkError
|
||||
var failed bool
|
||||
for i := range b.actions {
|
||||
action := &b.actions[i]
|
||||
var ok bool
|
||||
switch action.op {
|
||||
case bulkInsert:
|
||||
ok = b.runInsert(action, &result, &berr)
|
||||
case bulkUpdate:
|
||||
ok = b.runUpdate(action, &result, &berr)
|
||||
case bulkRemove:
|
||||
ok = b.runRemove(action, &result, &berr)
|
||||
default:
|
||||
panic("unknown bulk operation")
|
||||
}
|
||||
action.idxs = action.idxs[0:0]
|
||||
action.docs = action.docs[0:0]
|
||||
actionPool.Put(action)
|
||||
if !ok {
|
||||
failed = true
|
||||
if b.ordered {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
sort.Sort(bulkErrorCases(berr.ecases))
|
||||
return nil, &berr
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (b *Bulk) runInsert(action *bulkAction, result *BulkResult, berr *BulkError) bool {
|
||||
op := &insertOp{b.c.FullName, action.docs, 0}
|
||||
if !b.ordered {
|
||||
op.flags = 1 // ContinueOnError
|
||||
}
|
||||
lerr, err := b.c.writeOp(op, b.ordered)
|
||||
return b.checkSuccess(action, berr, lerr, err)
|
||||
}
|
||||
|
||||
func (b *Bulk) runUpdate(action *bulkAction, result *BulkResult, berr *BulkError) bool {
|
||||
lerr, err := b.c.writeOp(bulkUpdateOp(action.docs), b.ordered)
|
||||
if lerr != nil {
|
||||
result.Matched += lerr.N
|
||||
result.Modified += lerr.modified
|
||||
}
|
||||
return b.checkSuccess(action, berr, lerr, err)
|
||||
}
|
||||
|
||||
func (b *Bulk) runRemove(action *bulkAction, result *BulkResult, berr *BulkError) bool {
|
||||
lerr, err := b.c.writeOp(bulkDeleteOp(action.docs), b.ordered)
|
||||
if lerr != nil {
|
||||
result.Matched += lerr.N
|
||||
result.Modified += lerr.modified
|
||||
}
|
||||
return b.checkSuccess(action, berr, lerr, err)
|
||||
}
|
||||
|
||||
func (b *Bulk) checkSuccess(action *bulkAction, berr *BulkError, lerr *LastError, err error) bool {
|
||||
if lerr != nil && len(lerr.ecases) > 0 {
|
||||
for i := 0; i < len(lerr.ecases); i++ {
|
||||
// Map back from the local error index into the visible one.
|
||||
ecase := lerr.ecases[i]
|
||||
idx := ecase.Index
|
||||
if idx >= 0 {
|
||||
idx = action.idxs[idx]
|
||||
}
|
||||
berr.ecases = append(berr.ecases, BulkErrorCase{idx, ecase.Err})
|
||||
}
|
||||
return false
|
||||
} else if err != nil {
|
||||
for i := 0; i < len(action.idxs); i++ {
|
||||
berr.ecases = append(berr.ecases, BulkErrorCase{action.idxs[i], err})
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
-357
@@ -1,357 +0,0 @@
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
type FullDocument string
|
||||
|
||||
const (
|
||||
Default = "default"
|
||||
UpdateLookup = "updateLookup"
|
||||
)
|
||||
|
||||
type ChangeStream struct {
|
||||
iter *Iter
|
||||
isClosed bool
|
||||
options ChangeStreamOptions
|
||||
pipeline interface{}
|
||||
resumeToken *bson.Raw
|
||||
collection *Collection
|
||||
readPreference *ReadPreference
|
||||
err error
|
||||
m sync.Mutex
|
||||
sessionCopied bool
|
||||
}
|
||||
|
||||
type ChangeStreamOptions struct {
|
||||
|
||||
// FullDocument controls the amount of data that the server will return when
|
||||
// returning a changes document.
|
||||
FullDocument FullDocument
|
||||
|
||||
// ResumeAfter specifies the logical starting point for the new change stream.
|
||||
ResumeAfter *bson.Raw
|
||||
|
||||
// MaxAwaitTimeMS specifies the maximum amount of time for the server to wait
|
||||
// on new documents to satisfy a change stream query.
|
||||
MaxAwaitTimeMS time.Duration
|
||||
|
||||
// BatchSize specifies the number of documents to return per batch.
|
||||
BatchSize int
|
||||
|
||||
// Collation specifies the way the server should collate returned data.
|
||||
//TODO Collation *Collation
|
||||
}
|
||||
|
||||
var errMissingResumeToken = errors.New("resume token missing from result")
|
||||
|
||||
// Watch constructs a new ChangeStream capable of receiving continuing data
|
||||
// from the database.
|
||||
func (coll *Collection) Watch(pipeline interface{},
|
||||
options ChangeStreamOptions) (*ChangeStream, error) {
|
||||
|
||||
if pipeline == nil {
|
||||
pipeline = []bson.M{}
|
||||
}
|
||||
|
||||
csPipe := constructChangeStreamPipeline(pipeline, options)
|
||||
pipe := coll.Pipe(&csPipe)
|
||||
if options.MaxAwaitTimeMS > 0 {
|
||||
pipe.SetMaxTime(options.MaxAwaitTimeMS)
|
||||
}
|
||||
if options.BatchSize > 0 {
|
||||
pipe.Batch(options.BatchSize)
|
||||
}
|
||||
pIter := pipe.Iter()
|
||||
|
||||
// check that there was no issue creating the iterator.
|
||||
// this will fail immediately with an error from the server if running against
|
||||
// a standalone.
|
||||
if err := pIter.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pIter.isChangeStream = true
|
||||
return &ChangeStream{
|
||||
iter: pIter,
|
||||
collection: coll,
|
||||
resumeToken: nil,
|
||||
options: options,
|
||||
pipeline: pipeline,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Next retrieves the next document from the change stream, blocking if necessary.
|
||||
// Next returns true if a document was successfully unmarshalled into result,
|
||||
// and false if an error occured. When Next returns false, the Err method should
|
||||
// be called to check what error occurred during iteration. If there were no events
|
||||
// available (ErrNotFound), the Err method returns nil so the user can retry the invocaton.
|
||||
//
|
||||
// For example:
|
||||
//
|
||||
// pipeline := []bson.M{}
|
||||
//
|
||||
// changeStream := collection.Watch(pipeline, ChangeStreamOptions{})
|
||||
// for changeStream.Next(&changeDoc) {
|
||||
// fmt.Printf("Change: %v\n", changeDoc)
|
||||
// }
|
||||
//
|
||||
// if err := changeStream.Close(); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// If the pipeline used removes the _id field from the result, Next will error
|
||||
// because the _id field is needed to resume iteration when an error occurs.
|
||||
//
|
||||
func (changeStream *ChangeStream) Next(result interface{}) bool {
|
||||
// the err field is being constantly overwritten and we don't want the user to
|
||||
// attempt to read it at this point so we lock.
|
||||
changeStream.m.Lock()
|
||||
|
||||
defer changeStream.m.Unlock()
|
||||
|
||||
// if we are in a state of error, then don't continue.
|
||||
if changeStream.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if changeStream.isClosed {
|
||||
changeStream.err = fmt.Errorf("illegal use of a closed ChangeStream")
|
||||
return false
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// attempt to fetch the change stream result.
|
||||
err = changeStream.fetchResultSet(result)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// if we get no results we return false with no errors so the user can call Next
|
||||
// again, resuming is not needed as the iterator is simply timed out as no events happened.
|
||||
// The user will call Timeout in order to understand if this was the case.
|
||||
if err == ErrNotFound {
|
||||
return false
|
||||
}
|
||||
|
||||
// check if the error is resumable
|
||||
if !isResumableError(err) {
|
||||
// error is not resumable, give up and return it to the user.
|
||||
changeStream.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
// try to resume.
|
||||
err = changeStream.resume()
|
||||
if err != nil {
|
||||
// we've not been able to successfully resume and should only try once,
|
||||
// so we give up.
|
||||
changeStream.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
// we've successfully resumed the changestream.
|
||||
// try to fetch the next result.
|
||||
err = changeStream.fetchResultSet(result)
|
||||
if err != nil {
|
||||
changeStream.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Err returns nil if no errors happened during iteration, or the actual
|
||||
// error otherwise.
|
||||
func (changeStream *ChangeStream) Err() error {
|
||||
changeStream.m.Lock()
|
||||
defer changeStream.m.Unlock()
|
||||
return changeStream.err
|
||||
}
|
||||
|
||||
// Close kills the server cursor used by the iterator, if any, and returns
|
||||
// nil if no errors happened during iteration, or the actual error otherwise.
|
||||
func (changeStream *ChangeStream) Close() error {
|
||||
changeStream.m.Lock()
|
||||
defer changeStream.m.Unlock()
|
||||
changeStream.isClosed = true
|
||||
err := changeStream.iter.Close()
|
||||
if err != nil {
|
||||
changeStream.err = err
|
||||
}
|
||||
if changeStream.sessionCopied {
|
||||
changeStream.iter.session.Close()
|
||||
changeStream.sessionCopied = false
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// ResumeToken returns a copy of the current resume token held by the change stream.
|
||||
// This token should be treated as an opaque token that can be provided to instantiate
|
||||
// a new change stream.
|
||||
func (changeStream *ChangeStream) ResumeToken() *bson.Raw {
|
||||
changeStream.m.Lock()
|
||||
defer changeStream.m.Unlock()
|
||||
if changeStream.resumeToken == nil {
|
||||
return nil
|
||||
}
|
||||
var tokenCopy = *changeStream.resumeToken
|
||||
return &tokenCopy
|
||||
}
|
||||
|
||||
// Timeout returns true if the last call of Next returned false because of an iterator timeout.
|
||||
func (changeStream *ChangeStream) Timeout() bool {
|
||||
return changeStream.iter.Timeout()
|
||||
}
|
||||
|
||||
func constructChangeStreamPipeline(pipeline interface{},
|
||||
options ChangeStreamOptions) interface{} {
|
||||
pipelinev := reflect.ValueOf(pipeline)
|
||||
|
||||
// ensure that the pipeline passed in is a slice.
|
||||
if pipelinev.Kind() != reflect.Slice {
|
||||
panic("pipeline argument must be a slice")
|
||||
}
|
||||
|
||||
// construct the options to be used by the change notification
|
||||
// pipeline stage.
|
||||
changeStreamStageOptions := bson.M{}
|
||||
|
||||
if options.FullDocument != "" {
|
||||
changeStreamStageOptions["fullDocument"] = options.FullDocument
|
||||
}
|
||||
if options.ResumeAfter != nil {
|
||||
changeStreamStageOptions["resumeAfter"] = options.ResumeAfter
|
||||
}
|
||||
|
||||
changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions}
|
||||
|
||||
pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1)
|
||||
|
||||
// insert the change notification pipeline stage at the beginning of the
|
||||
// aggregation.
|
||||
pipeOfInterfaces[0] = changeStreamStage
|
||||
|
||||
// convert the passed in slice to a slice of interfaces.
|
||||
for i := 0; i < pipelinev.Len(); i++ {
|
||||
pipeOfInterfaces[1+i] = pipelinev.Index(i).Addr().Interface()
|
||||
}
|
||||
var pipelineAsInterface interface{} = pipeOfInterfaces
|
||||
return pipelineAsInterface
|
||||
}
|
||||
|
||||
func (changeStream *ChangeStream) resume() error {
|
||||
// copy the information for the new socket.
|
||||
|
||||
// Thanks to Copy() future uses will acquire a new socket against the newly selected DB.
|
||||
newSession := changeStream.iter.session.Copy()
|
||||
|
||||
// fetch the cursor from the iterator and use it to run a killCursors
|
||||
// on the connection.
|
||||
cursorId := changeStream.iter.op.cursorId
|
||||
err := runKillCursorsOnSession(newSession, cursorId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// change out the old connection to the database with the new connection.
|
||||
if changeStream.sessionCopied {
|
||||
changeStream.collection.Database.Session.Close()
|
||||
}
|
||||
changeStream.collection.Database.Session = newSession
|
||||
changeStream.sessionCopied = true
|
||||
|
||||
opts := changeStream.options
|
||||
if changeStream.resumeToken != nil {
|
||||
opts.ResumeAfter = changeStream.resumeToken
|
||||
}
|
||||
// make a new pipeline containing the resume token.
|
||||
changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, opts)
|
||||
|
||||
// generate the new iterator with the new connection.
|
||||
newPipe := changeStream.collection.Pipe(changeStreamPipeline)
|
||||
changeStream.iter = newPipe.Iter()
|
||||
if err := changeStream.iter.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
changeStream.iter.isChangeStream = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchResumeToken unmarshals the _id field from the document, setting an error
|
||||
// on the changeStream if it is unable to.
|
||||
func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error {
|
||||
changeStreamResult := struct {
|
||||
ResumeToken *bson.Raw `bson:"_id,omitempty"`
|
||||
}{}
|
||||
|
||||
err := rawResult.Unmarshal(&changeStreamResult)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if changeStreamResult.ResumeToken == nil {
|
||||
return errMissingResumeToken
|
||||
}
|
||||
|
||||
changeStream.resumeToken = changeStreamResult.ResumeToken
|
||||
return nil
|
||||
}
|
||||
|
||||
func (changeStream *ChangeStream) fetchResultSet(result interface{}) error {
|
||||
rawResult := bson.Raw{}
|
||||
|
||||
// fetch the next set of documents from the cursor.
|
||||
gotNext := changeStream.iter.Next(&rawResult)
|
||||
err := changeStream.iter.Err()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !gotNext && err == nil {
|
||||
// If the iter.Err() method returns nil despite us not getting a next batch,
|
||||
// it is becuase iter.Err() silences this case.
|
||||
return ErrNotFound
|
||||
}
|
||||
|
||||
// grab the resumeToken from the results
|
||||
if err := changeStream.fetchResumeToken(&rawResult); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// put the raw results into the data structure the user provided.
|
||||
if err := rawResult.Unmarshal(result); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isResumableError(err error) bool {
|
||||
_, isQueryError := err.(*QueryError)
|
||||
// if it is not a database error OR it is a database error,
|
||||
// but the error is a notMaster error
|
||||
//and is not a missingResumeToken error (caused by the user provided pipeline)
|
||||
return (!isQueryError || isNotMasterError(err)) && (err != errMissingResumeToken)
|
||||
}
|
||||
|
||||
func runKillCursorsOnSession(session *Session, cursorId int64) error {
|
||||
socket, err := session.acquireSocket(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = socket.Query(&killCursorsOp{[]int64{cursorId}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
socket.Release()
|
||||
|
||||
return nil
|
||||
}
|
||||
-704
@@ -1,704 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mongo cluster encapsulation.
|
||||
//
|
||||
// A cluster enables the communication with one or more servers participating
|
||||
// in a mongo cluster. This works with individual servers, a replica set,
|
||||
// a replica pair, one or multiple mongos routers, etc.
|
||||
|
||||
type mongoCluster struct {
|
||||
sync.RWMutex
|
||||
serverSynced sync.Cond
|
||||
userSeeds []string
|
||||
dynaSeeds []string
|
||||
servers mongoServers
|
||||
masters mongoServers
|
||||
references int
|
||||
syncing bool
|
||||
syncCount uint
|
||||
cachedIndex map[string]bool
|
||||
sync chan bool
|
||||
dial dialer
|
||||
dialInfo *DialInfo
|
||||
}
|
||||
|
||||
func newCluster(userSeeds []string, info *DialInfo) *mongoCluster {
|
||||
cluster := &mongoCluster{
|
||||
userSeeds: userSeeds,
|
||||
references: 1,
|
||||
dial: dialer{info.Dial, info.DialServer},
|
||||
dialInfo: info,
|
||||
}
|
||||
cluster.serverSynced.L = cluster.RWMutex.RLocker()
|
||||
cluster.sync = make(chan bool, 1)
|
||||
stats.cluster(+1)
|
||||
go cluster.syncServersLoop()
|
||||
return cluster
|
||||
}
|
||||
|
||||
// Acquire increases the reference count for the cluster.
|
||||
func (cluster *mongoCluster) Acquire() {
|
||||
cluster.Lock()
|
||||
cluster.references++
|
||||
debugf("Cluster %p acquired (refs=%d)", cluster, cluster.references)
|
||||
cluster.Unlock()
|
||||
}
|
||||
|
||||
// Release decreases the reference count for the cluster. Once
|
||||
// it reaches zero, all servers will be closed.
|
||||
func (cluster *mongoCluster) Release() {
|
||||
cluster.Lock()
|
||||
if cluster.references == 0 {
|
||||
panic("cluster.Release() with references == 0")
|
||||
}
|
||||
cluster.references--
|
||||
debugf("Cluster %p released (refs=%d)", cluster, cluster.references)
|
||||
if cluster.references == 0 {
|
||||
for _, server := range cluster.servers.Slice() {
|
||||
server.Close()
|
||||
}
|
||||
// Wake up the sync loop so it can die.
|
||||
cluster.syncServers()
|
||||
stats.cluster(-1)
|
||||
}
|
||||
cluster.Unlock()
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) LiveServers() (servers []string) {
|
||||
cluster.RLock()
|
||||
for _, serv := range cluster.servers.Slice() {
|
||||
servers = append(servers, serv.Addr)
|
||||
}
|
||||
cluster.RUnlock()
|
||||
return servers
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) removeServer(server *mongoServer) {
|
||||
cluster.Lock()
|
||||
cluster.masters.Remove(server)
|
||||
other := cluster.servers.Remove(server)
|
||||
cluster.Unlock()
|
||||
if other != nil {
|
||||
other.CloseIdle()
|
||||
log("Removed server ", server.Addr, " from cluster.")
|
||||
}
|
||||
server.CloseIdle()
|
||||
}
|
||||
|
||||
type isMasterResult struct {
|
||||
IsMaster bool
|
||||
Secondary bool
|
||||
Primary string
|
||||
Hosts []string
|
||||
Passives []string
|
||||
Tags bson.D
|
||||
Msg string
|
||||
SetName string `bson:"setName"`
|
||||
MaxWireVersion int `bson:"maxWireVersion"`
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResult) error {
|
||||
// Monotonic let's it talk to a slave and still hold the socket.
|
||||
session := newSession(Monotonic, cluster, cluster.dialInfo)
|
||||
session.setSocket(socket)
|
||||
|
||||
var cmd = bson.D{{Name: "isMaster", Value: 1}}
|
||||
|
||||
// Send client metadata to the server to identify this socket if this is
|
||||
// the first isMaster call only.
|
||||
//
|
||||
// isMaster commands issued after the initial connection handshake MUST NOT contain handshake arguments
|
||||
// https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#connection-handshake
|
||||
//
|
||||
socket.sendMeta.Do(func() {
|
||||
var meta = bson.M{
|
||||
"driver": bson.M{
|
||||
"name": "mgo",
|
||||
"version": "globalsign",
|
||||
},
|
||||
"os": bson.M{
|
||||
"type": runtime.GOOS,
|
||||
"architecture": runtime.GOARCH,
|
||||
},
|
||||
}
|
||||
|
||||
// Include the application name if set
|
||||
if cluster.dialInfo.AppName != "" {
|
||||
meta["application"] = bson.M{"name": cluster.dialInfo.AppName}
|
||||
}
|
||||
|
||||
cmd = append(cmd, bson.DocElem{
|
||||
Name: "client",
|
||||
Value: meta,
|
||||
})
|
||||
})
|
||||
|
||||
err := session.runOnSocket(socket, cmd, result)
|
||||
session.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
type possibleTimeout interface {
|
||||
Timeout() bool
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) syncServer(server *mongoServer) (info *mongoServerInfo, hosts []string, err error) {
|
||||
addr := server.Addr
|
||||
log("SYNC Processing ", addr, "...")
|
||||
|
||||
// Retry a few times to avoid knocking a server down for a hiccup.
|
||||
var result isMasterResult
|
||||
var tryerr error
|
||||
for retry := 0; ; retry++ {
|
||||
if retry == 3 || retry == 1 && cluster.dialInfo.FailFast {
|
||||
return nil, nil, tryerr
|
||||
}
|
||||
if retry > 0 {
|
||||
// Don't abuse the server needlessly if there's something actually wrong.
|
||||
if err, ok := tryerr.(possibleTimeout); ok && err.Timeout() {
|
||||
// Give a chance for waiters to timeout as well.
|
||||
cluster.serverSynced.Broadcast()
|
||||
}
|
||||
time.Sleep(syncShortDelay)
|
||||
}
|
||||
|
||||
// Don't ever hit the pool limit for syncing
|
||||
config := cluster.dialInfo.Copy()
|
||||
config.PoolLimit = 0
|
||||
|
||||
socket, _, err := server.AcquireSocket(config)
|
||||
if err != nil {
|
||||
tryerr = err
|
||||
logf("SYNC Failed to get socket to %s: %v", addr, err)
|
||||
continue
|
||||
}
|
||||
err = cluster.isMaster(socket, &result)
|
||||
|
||||
// Restore the correct dial config before returning it to the pool
|
||||
socket.dialInfo = cluster.dialInfo
|
||||
socket.Release()
|
||||
|
||||
if err != nil {
|
||||
tryerr = err
|
||||
logf("SYNC Command 'ismaster' to %s failed: %v", addr, err)
|
||||
continue
|
||||
}
|
||||
debugf("SYNC Result of 'ismaster' from %s: %#v", addr, result)
|
||||
break
|
||||
}
|
||||
|
||||
if cluster.dialInfo.ReplicaSetName != "" && result.SetName != cluster.dialInfo.ReplicaSetName {
|
||||
logf("SYNC Server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName)
|
||||
return nil, nil, fmt.Errorf("server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName)
|
||||
}
|
||||
|
||||
if result.IsMaster {
|
||||
debugf("SYNC %s is a master.", addr)
|
||||
if !server.info.Master {
|
||||
// Made an incorrect assumption above, so fix stats.
|
||||
stats.conn(-1, false)
|
||||
stats.conn(+1, true)
|
||||
}
|
||||
} else if result.Secondary {
|
||||
debugf("SYNC %s is a slave.", addr)
|
||||
} else if cluster.dialInfo.Direct {
|
||||
logf("SYNC %s in unknown state. Pretending it's a slave due to direct connection.", addr)
|
||||
} else {
|
||||
logf("SYNC %s is neither a master nor a slave.", addr)
|
||||
// Let stats track it as whatever was known before.
|
||||
return nil, nil, errors.New(addr + " is not a master nor slave")
|
||||
}
|
||||
|
||||
info = &mongoServerInfo{
|
||||
Master: result.IsMaster,
|
||||
Mongos: result.Msg == "isdbgrid",
|
||||
Tags: result.Tags,
|
||||
SetName: result.SetName,
|
||||
MaxWireVersion: result.MaxWireVersion,
|
||||
}
|
||||
|
||||
hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives))
|
||||
if result.Primary != "" {
|
||||
// First in the list to speed up master discovery.
|
||||
hosts = append(hosts, result.Primary)
|
||||
}
|
||||
hosts = append(hosts, result.Hosts...)
|
||||
hosts = append(hosts, result.Passives...)
|
||||
|
||||
debugf("SYNC %s knows about the following peers: %#v", addr, hosts)
|
||||
return info, hosts, nil
|
||||
}
|
||||
|
||||
type syncKind bool
|
||||
|
||||
const (
|
||||
completeSync syncKind = true
|
||||
partialSync syncKind = false
|
||||
)
|
||||
|
||||
func (cluster *mongoCluster) addServer(server *mongoServer, info *mongoServerInfo, syncKind syncKind) {
|
||||
cluster.Lock()
|
||||
current := cluster.servers.Search(server.ResolvedAddr)
|
||||
if current == nil {
|
||||
if syncKind == partialSync {
|
||||
cluster.Unlock()
|
||||
server.Close()
|
||||
log("SYNC Discarding unknown server ", server.Addr, " due to partial sync.")
|
||||
return
|
||||
}
|
||||
cluster.servers.Add(server)
|
||||
if info.Master {
|
||||
cluster.masters.Add(server)
|
||||
log("SYNC Adding ", server.Addr, " to cluster as a master.")
|
||||
} else {
|
||||
log("SYNC Adding ", server.Addr, " to cluster as a slave.")
|
||||
}
|
||||
} else {
|
||||
if server != current {
|
||||
panic("addServer attempting to add duplicated server")
|
||||
}
|
||||
if server.Info().Master != info.Master {
|
||||
if info.Master {
|
||||
log("SYNC Server ", server.Addr, " is now a master.")
|
||||
cluster.masters.Add(server)
|
||||
} else {
|
||||
log("SYNC Server ", server.Addr, " is now a slave.")
|
||||
cluster.masters.Remove(server)
|
||||
}
|
||||
}
|
||||
}
|
||||
server.SetInfo(info)
|
||||
debugf("SYNC Broadcasting availability of server %s", server.Addr)
|
||||
cluster.serverSynced.Broadcast()
|
||||
cluster.Unlock()
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) getKnownAddrs() []string {
|
||||
cluster.RLock()
|
||||
max := len(cluster.userSeeds) + len(cluster.dynaSeeds) + cluster.servers.Len()
|
||||
seen := make(map[string]bool, max)
|
||||
known := make([]string, 0, max)
|
||||
|
||||
add := func(addr string) {
|
||||
if _, found := seen[addr]; !found {
|
||||
seen[addr] = true
|
||||
known = append(known, addr)
|
||||
}
|
||||
}
|
||||
|
||||
for _, addr := range cluster.userSeeds {
|
||||
add(addr)
|
||||
}
|
||||
for _, addr := range cluster.dynaSeeds {
|
||||
add(addr)
|
||||
}
|
||||
for _, serv := range cluster.servers.Slice() {
|
||||
add(serv.Addr)
|
||||
}
|
||||
cluster.RUnlock()
|
||||
|
||||
return known
|
||||
}
|
||||
|
||||
// syncServers injects a value into the cluster.sync channel to force
|
||||
// an iteration of the syncServersLoop function.
|
||||
func (cluster *mongoCluster) syncServers() {
|
||||
select {
|
||||
case cluster.sync <- true:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// How long to wait for a checkup of the cluster topology if nothing
|
||||
// else kicks a synchronization before that.
|
||||
const syncServersDelay = 30 * time.Second
|
||||
const syncShortDelay = 500 * time.Millisecond
|
||||
|
||||
// syncServersLoop loops while the cluster is alive to keep its idea of
|
||||
// the server topology up-to-date. It must be called just once from
|
||||
// newCluster. The loop iterates once syncServersDelay has passed, or
|
||||
// if somebody injects a value into the cluster.sync channel to force a
|
||||
// synchronization. A loop iteration will contact all servers in
|
||||
// parallel, ask them about known peers and their own role within the
|
||||
// cluster, and then attempt to do the same with all the peers
|
||||
// retrieved.
|
||||
func (cluster *mongoCluster) syncServersLoop() {
|
||||
for {
|
||||
debugf("SYNC Cluster %p is starting a sync loop iteration.", cluster)
|
||||
|
||||
cluster.Lock()
|
||||
if cluster.references == 0 {
|
||||
cluster.Unlock()
|
||||
break
|
||||
}
|
||||
cluster.references++ // Keep alive while syncing.
|
||||
direct := cluster.dialInfo.Direct
|
||||
cluster.Unlock()
|
||||
|
||||
cluster.syncServersIteration(direct)
|
||||
|
||||
// We just synchronized, so consume any outstanding requests.
|
||||
select {
|
||||
case <-cluster.sync:
|
||||
default:
|
||||
}
|
||||
|
||||
cluster.Release()
|
||||
|
||||
// Hold off before allowing another sync. No point in
|
||||
// burning CPU looking for down servers.
|
||||
if !cluster.dialInfo.FailFast {
|
||||
time.Sleep(syncShortDelay)
|
||||
}
|
||||
|
||||
cluster.Lock()
|
||||
if cluster.references == 0 {
|
||||
cluster.Unlock()
|
||||
break
|
||||
}
|
||||
cluster.syncCount++
|
||||
// Poke all waiters so they have a chance to timeout or
|
||||
// restart syncing if they wish to.
|
||||
cluster.serverSynced.Broadcast()
|
||||
// Check if we have to restart immediately either way.
|
||||
restart := !direct && cluster.masters.Empty() || cluster.servers.Empty()
|
||||
cluster.Unlock()
|
||||
|
||||
if restart {
|
||||
log("SYNC No masters found. Will synchronize again.")
|
||||
time.Sleep(syncShortDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
debugf("SYNC Cluster %p waiting for next requested or scheduled sync.", cluster)
|
||||
|
||||
// Hold off until somebody explicitly requests a synchronization
|
||||
// or it's time to check for a cluster topology change again.
|
||||
select {
|
||||
case <-cluster.sync:
|
||||
case <-time.After(syncServersDelay):
|
||||
}
|
||||
}
|
||||
debugf("SYNC Cluster %p is stopping its sync loop.", cluster)
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) server(addr string, tcpaddr *net.TCPAddr) *mongoServer {
|
||||
cluster.RLock()
|
||||
server := cluster.servers.Search(tcpaddr.String())
|
||||
cluster.RUnlock()
|
||||
if server != nil {
|
||||
return server
|
||||
}
|
||||
return newServer(addr, tcpaddr, cluster.sync, cluster.dial, cluster.dialInfo)
|
||||
}
|
||||
|
||||
func resolveAddr(addr string) (*net.TCPAddr, error) {
|
||||
// Simple cases that do not need actual resolution. Works with IPv4 and v6.
|
||||
if host, port, err := net.SplitHostPort(addr); err == nil {
|
||||
if port, _ := strconv.Atoi(port); port > 0 {
|
||||
zone := ""
|
||||
if i := strings.LastIndex(host, "%"); i >= 0 {
|
||||
zone = host[i+1:]
|
||||
host = host[:i]
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip != nil {
|
||||
return &net.TCPAddr{IP: ip, Port: port, Zone: zone}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to resolve IPv4 and v6 concurrently.
|
||||
addrChan := make(chan *net.TCPAddr, 2)
|
||||
for _, network := range []string{"udp4", "udp6"} {
|
||||
network := network
|
||||
go func() {
|
||||
// The unfortunate UDP dialing hack allows having a timeout on address resolution.
|
||||
conn, err := net.DialTimeout(network, addr, 10*time.Second)
|
||||
if err != nil {
|
||||
addrChan <- nil
|
||||
} else {
|
||||
addrChan <- (*net.TCPAddr)(conn.RemoteAddr().(*net.UDPAddr))
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for the result of IPv4 and v6 resolution. Use IPv4 if available.
|
||||
tcpaddr := <-addrChan
|
||||
if tcpaddr == nil || len(tcpaddr.IP) != 4 {
|
||||
var timeout <-chan time.Time
|
||||
if tcpaddr != nil {
|
||||
// Don't wait too long if an IPv6 address is known.
|
||||
timeout = time.After(50 * time.Millisecond)
|
||||
}
|
||||
select {
|
||||
case <-timeout:
|
||||
case tcpaddr2 := <-addrChan:
|
||||
if tcpaddr == nil || tcpaddr2 != nil {
|
||||
// It's an IPv4 address or the only known address. Use it.
|
||||
tcpaddr = tcpaddr2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if tcpaddr == nil {
|
||||
log("SYNC Failed to resolve server address: ", addr)
|
||||
return nil, errors.New("failed to resolve server address: " + addr)
|
||||
}
|
||||
if tcpaddr.String() != addr {
|
||||
debug("SYNC Address ", addr, " resolved as ", tcpaddr.String())
|
||||
}
|
||||
return tcpaddr, nil
|
||||
}
|
||||
|
||||
type pendingAdd struct {
|
||||
server *mongoServer
|
||||
info *mongoServerInfo
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) syncServersIteration(direct bool) {
|
||||
log("SYNC Starting full topology synchronization...")
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var m sync.Mutex
|
||||
notYetAdded := make(map[string]pendingAdd)
|
||||
addIfFound := make(map[string]bool)
|
||||
seen := make(map[string]bool)
|
||||
syncKind := partialSync
|
||||
|
||||
var spawnSync func(addr string, byMaster bool)
|
||||
spawnSync = func(addr string, byMaster bool) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
tcpaddr, err := resolveAddr(addr)
|
||||
if err != nil {
|
||||
log("SYNC Failed to start sync of ", addr, ": ", err.Error())
|
||||
return
|
||||
}
|
||||
resolvedAddr := tcpaddr.String()
|
||||
|
||||
m.Lock()
|
||||
if byMaster {
|
||||
if pending, ok := notYetAdded[resolvedAddr]; ok {
|
||||
delete(notYetAdded, resolvedAddr)
|
||||
m.Unlock()
|
||||
cluster.addServer(pending.server, pending.info, completeSync)
|
||||
return
|
||||
}
|
||||
addIfFound[resolvedAddr] = true
|
||||
}
|
||||
if seen[resolvedAddr] {
|
||||
m.Unlock()
|
||||
return
|
||||
}
|
||||
seen[resolvedAddr] = true
|
||||
m.Unlock()
|
||||
|
||||
server := cluster.server(addr, tcpaddr)
|
||||
info, hosts, err := cluster.syncServer(server)
|
||||
if err != nil {
|
||||
cluster.removeServer(server)
|
||||
return
|
||||
}
|
||||
|
||||
m.Lock()
|
||||
add := direct || info.Master || addIfFound[resolvedAddr]
|
||||
if add {
|
||||
syncKind = completeSync
|
||||
} else {
|
||||
notYetAdded[resolvedAddr] = pendingAdd{server, info}
|
||||
}
|
||||
m.Unlock()
|
||||
if add {
|
||||
cluster.addServer(server, info, completeSync)
|
||||
}
|
||||
if !direct {
|
||||
for _, addr := range hosts {
|
||||
spawnSync(addr, info.Master)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
knownAddrs := cluster.getKnownAddrs()
|
||||
for _, addr := range knownAddrs {
|
||||
spawnSync(addr, false)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if syncKind == completeSync {
|
||||
logf("SYNC Synchronization was complete (got data from primary).")
|
||||
for _, pending := range notYetAdded {
|
||||
cluster.removeServer(pending.server)
|
||||
}
|
||||
} else {
|
||||
logf("SYNC Synchronization was partial (cannot talk to primary).")
|
||||
for _, pending := range notYetAdded {
|
||||
cluster.addServer(pending.server, pending.info, partialSync)
|
||||
}
|
||||
}
|
||||
|
||||
cluster.Lock()
|
||||
mastersLen := cluster.masters.Len()
|
||||
logf("SYNC Synchronization completed: %d master(s) and %d slave(s) alive.", mastersLen, cluster.servers.Len()-mastersLen)
|
||||
|
||||
// Update dynamic seeds, but only if we have any good servers. Otherwise,
|
||||
// leave them alone for better chances of a successful sync in the future.
|
||||
if syncKind == completeSync {
|
||||
dynaSeeds := make([]string, cluster.servers.Len())
|
||||
for i, server := range cluster.servers.Slice() {
|
||||
dynaSeeds[i] = server.Addr
|
||||
}
|
||||
cluster.dynaSeeds = dynaSeeds
|
||||
debugf("SYNC New dynamic seeds: %#v\n", dynaSeeds)
|
||||
}
|
||||
cluster.Unlock()
|
||||
}
|
||||
|
||||
// AcquireSocketWithPoolTimeout returns a socket to a server in the cluster. If slaveOk is
|
||||
// true, it will attempt to return a socket to a slave server. If it is
|
||||
// false, the socket will necessarily be to a master server.
|
||||
func (cluster *mongoCluster) AcquireSocketWithPoolTimeout(mode Mode, slaveOk bool, syncTimeout time.Duration, serverTags []bson.D, info *DialInfo) (s *mongoSocket, err error) {
|
||||
var started time.Time
|
||||
var syncCount uint
|
||||
for {
|
||||
cluster.RLock()
|
||||
for {
|
||||
mastersLen := cluster.masters.Len()
|
||||
slavesLen := cluster.servers.Len() - mastersLen
|
||||
debugf("Cluster has %d known masters and %d known slaves.", mastersLen, slavesLen)
|
||||
if mastersLen > 0 && !(slaveOk && mode == Secondary) || slavesLen > 0 && slaveOk {
|
||||
break
|
||||
}
|
||||
if mastersLen > 0 && mode == Secondary && cluster.masters.HasMongos() {
|
||||
break
|
||||
}
|
||||
if started.IsZero() {
|
||||
// Initialize after fast path above.
|
||||
started = time.Now()
|
||||
syncCount = cluster.syncCount
|
||||
} else if syncTimeout != 0 && started.Before(time.Now().Add(-syncTimeout)) || cluster.dialInfo.FailFast && cluster.syncCount != syncCount {
|
||||
cluster.RUnlock()
|
||||
return nil, errors.New("no reachable servers")
|
||||
}
|
||||
log("Waiting for servers to synchronize...")
|
||||
cluster.syncServers()
|
||||
|
||||
// Remember: this will release and reacquire the lock.
|
||||
cluster.serverSynced.Wait()
|
||||
}
|
||||
|
||||
var server *mongoServer
|
||||
if slaveOk {
|
||||
server = cluster.servers.BestFit(mode, serverTags)
|
||||
} else {
|
||||
server = cluster.masters.BestFit(mode, nil)
|
||||
}
|
||||
cluster.RUnlock()
|
||||
|
||||
if server == nil {
|
||||
// Must have failed the requested tags. Sleep to avoid spinning.
|
||||
time.Sleep(1e8)
|
||||
continue
|
||||
}
|
||||
|
||||
s, abended, err := server.AcquireSocketWithBlocking(info)
|
||||
if err == errPoolTimeout {
|
||||
// No need to remove servers from the topology if acquiring a socket fails for this reason.
|
||||
return nil, err
|
||||
}
|
||||
if err != nil {
|
||||
cluster.removeServer(server)
|
||||
cluster.syncServers()
|
||||
continue
|
||||
}
|
||||
if abended && !slaveOk {
|
||||
var result isMasterResult
|
||||
err := cluster.isMaster(s, &result)
|
||||
if err != nil || !result.IsMaster {
|
||||
logf("Cannot confirm server %s as master (%v)", server.Addr, err)
|
||||
s.Release()
|
||||
cluster.syncServers()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
} else {
|
||||
// We've managed to successfully reconnect to the master, we are no longer abnormaly ended
|
||||
server.Lock()
|
||||
server.abended = false
|
||||
server.Unlock()
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) {
|
||||
cluster.Lock()
|
||||
if cluster.cachedIndex == nil {
|
||||
cluster.cachedIndex = make(map[string]bool)
|
||||
}
|
||||
if exists {
|
||||
cluster.cachedIndex[cacheKey] = true
|
||||
} else {
|
||||
delete(cluster.cachedIndex, cacheKey)
|
||||
}
|
||||
cluster.Unlock()
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) HasCachedIndex(cacheKey string) (result bool) {
|
||||
cluster.RLock()
|
||||
if cluster.cachedIndex != nil {
|
||||
result = cluster.cachedIndex[cacheKey]
|
||||
}
|
||||
cluster.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *mongoCluster) ResetIndexCache() {
|
||||
cluster.Lock()
|
||||
cluster.cachedIndex = make(map[string]bool)
|
||||
cluster.Unlock()
|
||||
}
|
||||
-62
@@ -1,62 +0,0 @@
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// coarseTimeProvider provides a periodically updated (approximate) time value to
|
||||
// amortise the cost of frequent calls to time.Now.
|
||||
//
|
||||
// A read throughput increase of ~6% was measured when using coarseTimeProvider with the
|
||||
// high-precision event timer (HPET) on FreeBSD 11.1 and Go 1.10.1 after merging
|
||||
// #116.
|
||||
//
|
||||
// Calling Now returns a time.Time that is updated at the configured interval,
|
||||
// however due to scheduling the value may be marginally older than expected.
|
||||
//
|
||||
// coarseTimeProvider is safe for concurrent use.
|
||||
type coarseTimeProvider struct {
|
||||
once sync.Once
|
||||
stop chan struct{}
|
||||
last atomic.Value
|
||||
}
|
||||
|
||||
// Now returns the most recently acquired time.Time value.
|
||||
func (t *coarseTimeProvider) Now() time.Time {
|
||||
return t.last.Load().(time.Time)
|
||||
}
|
||||
|
||||
// Close stops the periodic update of t.
|
||||
//
|
||||
// Any subsequent calls to Now will return the same value forever.
|
||||
func (t *coarseTimeProvider) Close() {
|
||||
t.once.Do(func() {
|
||||
close(t.stop)
|
||||
})
|
||||
}
|
||||
|
||||
// newcoarseTimeProvider returns a coarseTimeProvider configured to update at granularity.
|
||||
func newcoarseTimeProvider(granularity time.Duration) *coarseTimeProvider {
|
||||
t := &coarseTimeProvider{
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
|
||||
t.last.Store(time.Now())
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(granularity)
|
||||
for {
|
||||
select {
|
||||
case <-t.stop:
|
||||
ticker.Stop()
|
||||
return
|
||||
case <-ticker.C:
|
||||
t.last.Store(time.Now())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return t
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
// Package mgo (pronounced as "mango") offers a rich MongoDB driver for Go.
|
||||
//
|
||||
// Detailed documentation of the API is available at GoDoc:
|
||||
//
|
||||
// https://godoc.org/github.com/globalsign/mgo
|
||||
//
|
||||
// Usage of the driver revolves around the concept of sessions. To
|
||||
// get started, obtain a session using the Dial function:
|
||||
//
|
||||
// session, err := mgo.Dial(url)
|
||||
//
|
||||
// This will establish one or more connections with the cluster of
|
||||
// servers defined by the url parameter. From then on, the cluster
|
||||
// may be queried with multiple consistency rules (see SetMode) and
|
||||
// documents retrieved with statements such as:
|
||||
//
|
||||
// c := session.DB(database).C(collection)
|
||||
// err := c.Find(query).One(&result)
|
||||
//
|
||||
// New sessions are typically created by calling session.Copy on the
|
||||
// initial session obtained at dial time. These new sessions will share
|
||||
// the same cluster information and connection pool, and may be easily
|
||||
// handed into other methods and functions for organizing logic.
|
||||
// Every session created must have its Close method called at the end
|
||||
// of its life time, so its resources may be put back in the pool or
|
||||
// collected, depending on the case.
|
||||
//
|
||||
// There is a sub-package that provides support for BSON, which can be
|
||||
// used by itself as well:
|
||||
//
|
||||
// https://godoc.org/github.com/globalsign/mgo/bson
|
||||
//
|
||||
// For more details, see the documentation for the types and methods.
|
||||
//
|
||||
package mgo
|
||||
-782
@@ -1,782 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
// GridFS stores files in two collections:
|
||||
//
|
||||
// - chunks stores the binary chunks. For details, see the chunks Collection.
|
||||
// - files stores the file’s metadata. For details, see the files Collection.
|
||||
//
|
||||
// GridFS places the collections in a common bucket by prefixing each with the bucket name.
|
||||
// By default, GridFS uses two collections with a bucket named fs:
|
||||
//
|
||||
// - fs.files
|
||||
// - fs.chunks
|
||||
//
|
||||
// You can choose a different bucket name, as well as create multiple buckets in a single database.
|
||||
// The full collection name, which includes the bucket name, is subject to the namespace length limit.
|
||||
//
|
||||
// Relevant documentation:
|
||||
//
|
||||
// https://docs.mongodb.com/manual/core/gridfs/
|
||||
// https://docs.mongodb.com/manual/core/gridfs/#gridfs-chunks-collection
|
||||
// https://docs.mongodb.com/manual/core/gridfs/#gridfs-files-collection
|
||||
//
|
||||
type GridFS struct {
|
||||
Files *Collection
|
||||
Chunks *Collection
|
||||
}
|
||||
|
||||
type gfsFileMode int
|
||||
|
||||
const (
|
||||
gfsClosed gfsFileMode = 0
|
||||
gfsReading gfsFileMode = 1
|
||||
gfsWriting gfsFileMode = 2
|
||||
)
|
||||
|
||||
// GridFile document in files collection
|
||||
type GridFile struct {
|
||||
m sync.Mutex
|
||||
c sync.Cond
|
||||
gfs *GridFS
|
||||
mode gfsFileMode
|
||||
err error
|
||||
|
||||
chunk int
|
||||
offset int64
|
||||
|
||||
wpending int
|
||||
wbuf []byte
|
||||
wsum hash.Hash
|
||||
|
||||
rbuf []byte
|
||||
rcache *gfsCachedChunk
|
||||
|
||||
doc gfsFile
|
||||
}
|
||||
|
||||
type gfsFile struct {
|
||||
Id interface{} `bson:"_id"`
|
||||
ChunkSize int `bson:"chunkSize"`
|
||||
UploadDate time.Time `bson:"uploadDate"`
|
||||
Length int64 `bson:",minsize"`
|
||||
MD5 string
|
||||
Filename string `bson:",omitempty"`
|
||||
ContentType string `bson:"contentType,omitempty"`
|
||||
Metadata *bson.Raw `bson:",omitempty"`
|
||||
}
|
||||
|
||||
type gfsChunk struct {
|
||||
Id interface{} `bson:"_id"`
|
||||
FilesId interface{} `bson:"files_id"`
|
||||
N int
|
||||
Data []byte
|
||||
}
|
||||
|
||||
type gfsCachedChunk struct {
|
||||
wait sync.Mutex
|
||||
n int
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func newGridFS(db *Database, prefix string) *GridFS {
|
||||
return &GridFS{db.C(prefix + ".files"), db.C(prefix + ".chunks")}
|
||||
}
|
||||
|
||||
func (gfs *GridFS) newFile() *GridFile {
|
||||
file := &GridFile{gfs: gfs}
|
||||
file.c.L = &file.m
|
||||
//runtime.SetFinalizer(file, finalizeFile)
|
||||
return file
|
||||
}
|
||||
|
||||
func finalizeFile(file *GridFile) {
|
||||
file.Close()
|
||||
}
|
||||
|
||||
// Create creates a new file with the provided name in the GridFS. If the file
|
||||
// name already exists, a new version will be inserted with an up-to-date
|
||||
// uploadDate that will cause it to be atomically visible to the Open and
|
||||
// OpenId methods. If the file name is not important, an empty name may be
|
||||
// provided and the file Id used instead.
|
||||
//
|
||||
// It's important to Close files whether they are being written to
|
||||
// or read from, and to check the err result to ensure the operation
|
||||
// completed successfully.
|
||||
//
|
||||
// A simple example inserting a new file:
|
||||
//
|
||||
// func check(err error) {
|
||||
// if err != nil {
|
||||
// panic(err.String())
|
||||
// }
|
||||
// }
|
||||
// file, err := db.GridFS("fs").Create("myfile.txt")
|
||||
// check(err)
|
||||
// n, err := file.Write([]byte("Hello world!"))
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
// fmt.Printf("%d bytes written\n", n)
|
||||
//
|
||||
// The io.Writer interface is implemented by *GridFile and may be used to
|
||||
// help on the file creation. For example:
|
||||
//
|
||||
// file, err := db.GridFS("fs").Create("myfile.txt")
|
||||
// check(err)
|
||||
// messages, err := os.Open("/var/log/messages")
|
||||
// check(err)
|
||||
// defer messages.Close()
|
||||
// err = io.Copy(file, messages)
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
//
|
||||
func (gfs *GridFS) Create(name string) (file *GridFile, err error) {
|
||||
file = gfs.newFile()
|
||||
file.mode = gfsWriting
|
||||
file.wsum = md5.New()
|
||||
file.doc = gfsFile{Id: bson.NewObjectId(), ChunkSize: 255 * 1024, Filename: name}
|
||||
return
|
||||
}
|
||||
|
||||
// OpenId returns the file with the provided id, for reading.
|
||||
// If the file isn't found, err will be set to mgo.ErrNotFound.
|
||||
//
|
||||
// It's important to Close files whether they are being written to
|
||||
// or read from, and to check the err result to ensure the operation
|
||||
// completed successfully.
|
||||
//
|
||||
// The following example will print the first 8192 bytes from the file:
|
||||
//
|
||||
// func check(err error) {
|
||||
// if err != nil {
|
||||
// panic(err.String())
|
||||
// }
|
||||
// }
|
||||
// file, err := db.GridFS("fs").OpenId(objid)
|
||||
// check(err)
|
||||
// b := make([]byte, 8192)
|
||||
// n, err := file.Read(b)
|
||||
// check(err)
|
||||
// fmt.Println(string(b))
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
// fmt.Printf("%d bytes read\n", n)
|
||||
//
|
||||
// The io.Reader interface is implemented by *GridFile and may be used to
|
||||
// deal with it. As an example, the following snippet will dump the whole
|
||||
// file into the standard output:
|
||||
//
|
||||
// file, err := db.GridFS("fs").OpenId(objid)
|
||||
// check(err)
|
||||
// err = io.Copy(os.Stdout, file)
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
//
|
||||
func (gfs *GridFS) OpenId(id interface{}) (file *GridFile, err error) {
|
||||
var doc gfsFile
|
||||
err = gfs.Files.Find(bson.M{"_id": id}).One(&doc)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
file = gfs.newFile()
|
||||
file.mode = gfsReading
|
||||
file.doc = doc
|
||||
return
|
||||
}
|
||||
|
||||
// Open returns the most recently uploaded file with the provided
|
||||
// name, for reading. If the file isn't found, err will be set
|
||||
// to mgo.ErrNotFound.
|
||||
//
|
||||
// It's important to Close files whether they are being written to
|
||||
// or read from, and to check the err result to ensure the operation
|
||||
// completed successfully.
|
||||
//
|
||||
// The following example will print the first 8192 bytes from the file:
|
||||
//
|
||||
// file, err := db.GridFS("fs").Open("myfile.txt")
|
||||
// check(err)
|
||||
// b := make([]byte, 8192)
|
||||
// n, err := file.Read(b)
|
||||
// check(err)
|
||||
// fmt.Println(string(b))
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
// fmt.Printf("%d bytes read\n", n)
|
||||
//
|
||||
// The io.Reader interface is implemented by *GridFile and may be used to
|
||||
// deal with it. As an example, the following snippet will dump the whole
|
||||
// file into the standard output:
|
||||
//
|
||||
// file, err := db.GridFS("fs").Open("myfile.txt")
|
||||
// check(err)
|
||||
// err = io.Copy(os.Stdout, file)
|
||||
// check(err)
|
||||
// err = file.Close()
|
||||
// check(err)
|
||||
//
|
||||
func (gfs *GridFS) Open(name string) (file *GridFile, err error) {
|
||||
var doc gfsFile
|
||||
err = gfs.Files.Find(bson.M{"filename": name}).Sort("-uploadDate").One(&doc)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
file = gfs.newFile()
|
||||
file.mode = gfsReading
|
||||
file.doc = doc
|
||||
return
|
||||
}
|
||||
|
||||
// OpenNext opens the next file from iter for reading, sets *file to it,
|
||||
// and returns true on the success case. If no more documents are available
|
||||
// on iter or an error occurred, *file is set to nil and the result is false.
|
||||
// Errors will be available via iter.Err().
|
||||
//
|
||||
// The iter parameter must be an iterator on the GridFS files collection.
|
||||
// Using the GridFS.Find method is an easy way to obtain such an iterator,
|
||||
// but any iterator on the collection will work.
|
||||
//
|
||||
// If the provided *file is non-nil, OpenNext will close it before attempting
|
||||
// to iterate to the next element. This means that in a loop one only
|
||||
// has to worry about closing files when breaking out of the loop early
|
||||
// (break, return, or panic).
|
||||
//
|
||||
// For example:
|
||||
//
|
||||
// gfs := db.GridFS("fs")
|
||||
// query := gfs.Find(nil).Sort("filename")
|
||||
// iter := query.Iter()
|
||||
// var f *mgo.GridFile
|
||||
// for gfs.OpenNext(iter, &f) {
|
||||
// fmt.Printf("Filename: %s\n", f.Name())
|
||||
// }
|
||||
// if iter.Close() != nil {
|
||||
// panic(iter.Close())
|
||||
// }
|
||||
//
|
||||
func (gfs *GridFS) OpenNext(iter *Iter, file **GridFile) bool {
|
||||
if *file != nil {
|
||||
// Ignoring the error here shouldn't be a big deal
|
||||
// as we're reading the file and the loop iteration
|
||||
// for this file is finished.
|
||||
_ = (*file).Close()
|
||||
}
|
||||
var doc gfsFile
|
||||
if !iter.Next(&doc) {
|
||||
*file = nil
|
||||
return false
|
||||
}
|
||||
f := gfs.newFile()
|
||||
f.mode = gfsReading
|
||||
f.doc = doc
|
||||
*file = f
|
||||
return true
|
||||
}
|
||||
|
||||
// Find runs query on GridFS's files collection and returns
|
||||
// the resulting Query.
|
||||
//
|
||||
// This logic:
|
||||
//
|
||||
// gfs := db.GridFS("fs")
|
||||
// iter := gfs.Find(nil).Iter()
|
||||
//
|
||||
// Is equivalent to:
|
||||
//
|
||||
// files := db.C("fs" + ".files")
|
||||
// iter := files.Find(nil).Iter()
|
||||
//
|
||||
func (gfs *GridFS) Find(query interface{}) *Query {
|
||||
return gfs.Files.Find(query)
|
||||
}
|
||||
|
||||
// RemoveId deletes the file with the provided id from the GridFS.
|
||||
func (gfs *GridFS) RemoveId(id interface{}) error {
|
||||
err := gfs.Files.Remove(bson.M{"_id": id})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: id}})
|
||||
return err
|
||||
}
|
||||
|
||||
type gfsDocId struct {
|
||||
Id interface{} `bson:"_id"`
|
||||
}
|
||||
|
||||
// Remove deletes all files with the provided name from the GridFS.
|
||||
func (gfs *GridFS) Remove(name string) (err error) {
|
||||
iter := gfs.Files.Find(bson.M{"filename": name}).Select(bson.M{"_id": 1}).Iter()
|
||||
var doc gfsDocId
|
||||
for iter.Next(&doc) {
|
||||
if e := gfs.RemoveId(doc.Id); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = iter.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (file *GridFile) assertMode(mode gfsFileMode) {
|
||||
switch file.mode {
|
||||
case mode:
|
||||
return
|
||||
case gfsWriting:
|
||||
panic("GridFile is open for writing")
|
||||
case gfsReading:
|
||||
panic("GridFile is open for reading")
|
||||
case gfsClosed:
|
||||
panic("GridFile is closed")
|
||||
default:
|
||||
panic("internal error: missing GridFile mode")
|
||||
}
|
||||
}
|
||||
|
||||
// SetChunkSize sets size of saved chunks. Once the file is written to, it
|
||||
// will be split in blocks of that size and each block saved into an
|
||||
// independent chunk document. The default chunk size is 255kb.
|
||||
//
|
||||
// It is a runtime error to call this function once the file has started
|
||||
// being written to.
|
||||
func (file *GridFile) SetChunkSize(bytes int) {
|
||||
file.assertMode(gfsWriting)
|
||||
debugf("GridFile %p: setting chunk size to %d", file, bytes)
|
||||
file.m.Lock()
|
||||
file.doc.ChunkSize = bytes
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// Id returns the current file Id.
|
||||
func (file *GridFile) Id() interface{} {
|
||||
return file.doc.Id
|
||||
}
|
||||
|
||||
// SetId changes the current file Id.
|
||||
//
|
||||
// It is a runtime error to call this function once the file has started
|
||||
// being written to, or when the file is not open for writing.
|
||||
func (file *GridFile) SetId(id interface{}) {
|
||||
file.assertMode(gfsWriting)
|
||||
file.m.Lock()
|
||||
file.doc.Id = id
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// Name returns the optional file name. An empty string will be returned
|
||||
// in case it is unset.
|
||||
func (file *GridFile) Name() string {
|
||||
return file.doc.Filename
|
||||
}
|
||||
|
||||
// SetName changes the optional file name. An empty string may be used to
|
||||
// unset it.
|
||||
//
|
||||
// It is a runtime error to call this function when the file is not open
|
||||
// for writing.
|
||||
func (file *GridFile) SetName(name string) {
|
||||
file.assertMode(gfsWriting)
|
||||
file.m.Lock()
|
||||
file.doc.Filename = name
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// ContentType returns the optional file content type. An empty string will be
|
||||
// returned in case it is unset.
|
||||
func (file *GridFile) ContentType() string {
|
||||
return file.doc.ContentType
|
||||
}
|
||||
|
||||
// SetContentType changes the optional file content type. An empty string may be
|
||||
// used to unset it.
|
||||
//
|
||||
// It is a runtime error to call this function when the file is not open
|
||||
// for writing.
|
||||
func (file *GridFile) SetContentType(ctype string) {
|
||||
file.assertMode(gfsWriting)
|
||||
file.m.Lock()
|
||||
file.doc.ContentType = ctype
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// GetMeta unmarshals the optional "metadata" field associated with the
|
||||
// file into the result parameter. The meaning of keys under that field
|
||||
// is user-defined. For example:
|
||||
//
|
||||
// result := struct{ INode int }{}
|
||||
// err = file.GetMeta(&result)
|
||||
// if err != nil {
|
||||
// panic(err.String())
|
||||
// }
|
||||
// fmt.Printf("inode: %d\n", result.INode)
|
||||
//
|
||||
func (file *GridFile) GetMeta(result interface{}) (err error) {
|
||||
file.m.Lock()
|
||||
if file.doc.Metadata != nil {
|
||||
err = bson.Unmarshal(file.doc.Metadata.Data, result)
|
||||
}
|
||||
file.m.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// SetMeta changes the optional "metadata" field associated with the
|
||||
// file. The meaning of keys under that field is user-defined.
|
||||
// For example:
|
||||
//
|
||||
// file.SetMeta(bson.M{"inode": inode})
|
||||
//
|
||||
// It is a runtime error to call this function when the file is not open
|
||||
// for writing.
|
||||
func (file *GridFile) SetMeta(metadata interface{}) {
|
||||
file.assertMode(gfsWriting)
|
||||
data, err := bson.Marshal(metadata)
|
||||
file.m.Lock()
|
||||
if err != nil && file.err == nil {
|
||||
file.err = err
|
||||
} else {
|
||||
file.doc.Metadata = &bson.Raw{Data: data}
|
||||
}
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// Size returns the file size in bytes.
|
||||
func (file *GridFile) Size() (bytes int64) {
|
||||
file.m.Lock()
|
||||
bytes = file.doc.Length
|
||||
file.m.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// MD5 returns the file MD5 as a hex-encoded string.
|
||||
func (file *GridFile) MD5() (md5 string) {
|
||||
return file.doc.MD5
|
||||
}
|
||||
|
||||
// UploadDate returns the file upload time.
|
||||
func (file *GridFile) UploadDate() time.Time {
|
||||
return file.doc.UploadDate
|
||||
}
|
||||
|
||||
// SetUploadDate changes the file upload time.
|
||||
//
|
||||
// It is a runtime error to call this function when the file is not open
|
||||
// for writing.
|
||||
func (file *GridFile) SetUploadDate(t time.Time) {
|
||||
file.assertMode(gfsWriting)
|
||||
file.m.Lock()
|
||||
file.doc.UploadDate = t
|
||||
file.m.Unlock()
|
||||
}
|
||||
|
||||
// Close flushes any pending changes in case the file is being written
|
||||
// to, waits for any background operations to finish, and closes the file.
|
||||
//
|
||||
// It's important to Close files whether they are being written to
|
||||
// or read from, and to check the err result to ensure the operation
|
||||
// completed successfully.
|
||||
func (file *GridFile) Close() (err error) {
|
||||
file.m.Lock()
|
||||
defer file.m.Unlock()
|
||||
if file.mode == gfsWriting {
|
||||
if len(file.wbuf) > 0 && file.err == nil {
|
||||
file.insertChunk(file.wbuf)
|
||||
file.wbuf = file.wbuf[0:0]
|
||||
}
|
||||
file.completeWrite()
|
||||
} else if file.mode == gfsReading && file.rcache != nil {
|
||||
file.rcache.wait.Lock()
|
||||
file.rcache = nil
|
||||
}
|
||||
file.mode = gfsClosed
|
||||
debugf("GridFile %p: closed", file)
|
||||
return file.err
|
||||
}
|
||||
|
||||
func (file *GridFile) completeWrite() {
|
||||
for file.wpending > 0 {
|
||||
debugf("GridFile %p: waiting for %d pending chunks to complete file write", file, file.wpending)
|
||||
file.c.Wait()
|
||||
}
|
||||
if file.err == nil {
|
||||
hexsum := hex.EncodeToString(file.wsum.Sum(nil))
|
||||
if file.doc.UploadDate.IsZero() {
|
||||
file.doc.UploadDate = bson.Now()
|
||||
}
|
||||
file.doc.MD5 = hexsum
|
||||
file.err = file.gfs.Files.Insert(file.doc)
|
||||
}
|
||||
if file.err != nil {
|
||||
file.gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: file.doc.Id}})
|
||||
}
|
||||
if file.err == nil {
|
||||
index := Index{
|
||||
Key: []string{"files_id", "n"},
|
||||
Unique: true,
|
||||
}
|
||||
file.err = file.gfs.Chunks.EnsureIndex(index)
|
||||
}
|
||||
}
|
||||
|
||||
// Abort cancels an in-progress write, preventing the file from being
|
||||
// automically created and ensuring previously written chunks are
|
||||
// removed when the file is closed.
|
||||
//
|
||||
// It is a runtime error to call Abort when the file was not opened
|
||||
// for writing.
|
||||
func (file *GridFile) Abort() {
|
||||
if file.mode != gfsWriting {
|
||||
panic("file.Abort must be called on file opened for writing")
|
||||
}
|
||||
file.err = errors.New("write aborted")
|
||||
}
|
||||
|
||||
// Write writes the provided data to the file and returns the
|
||||
// number of bytes written and an error in case something
|
||||
// wrong happened.
|
||||
//
|
||||
// The file will internally cache the data so that all but the last
|
||||
// chunk sent to the database have the size defined by SetChunkSize.
|
||||
// This also means that errors may be deferred until a future call
|
||||
// to Write or Close.
|
||||
//
|
||||
// The parameters and behavior of this function turn the file
|
||||
// into an io.Writer.
|
||||
func (file *GridFile) Write(data []byte) (n int, err error) {
|
||||
file.assertMode(gfsWriting)
|
||||
file.m.Lock()
|
||||
debugf("GridFile %p: writing %d bytes", file, len(data))
|
||||
defer file.m.Unlock()
|
||||
|
||||
if file.err != nil {
|
||||
return 0, file.err
|
||||
}
|
||||
|
||||
n = len(data)
|
||||
file.doc.Length += int64(n)
|
||||
chunkSize := file.doc.ChunkSize
|
||||
|
||||
if len(file.wbuf)+len(data) < chunkSize {
|
||||
file.wbuf = append(file.wbuf, data...)
|
||||
return
|
||||
}
|
||||
|
||||
// First, flush file.wbuf complementing with data.
|
||||
if len(file.wbuf) > 0 {
|
||||
missing := chunkSize - len(file.wbuf)
|
||||
if missing > len(data) {
|
||||
missing = len(data)
|
||||
}
|
||||
file.wbuf = append(file.wbuf, data[:missing]...)
|
||||
data = data[missing:]
|
||||
file.insertChunk(file.wbuf)
|
||||
file.wbuf = file.wbuf[0:0]
|
||||
}
|
||||
|
||||
// Then, flush all chunks from data without copying.
|
||||
for len(data) > chunkSize {
|
||||
size := chunkSize
|
||||
if size > len(data) {
|
||||
size = len(data)
|
||||
}
|
||||
file.insertChunk(data[:size])
|
||||
data = data[size:]
|
||||
}
|
||||
|
||||
// And append the rest for a future call.
|
||||
file.wbuf = append(file.wbuf, data...)
|
||||
|
||||
return n, file.err
|
||||
}
|
||||
|
||||
func (file *GridFile) insertChunk(data []byte) {
|
||||
n := file.chunk
|
||||
file.chunk++
|
||||
debugf("GridFile %p: adding to checksum: %q", file, string(data))
|
||||
file.wsum.Write(data)
|
||||
|
||||
for file.doc.ChunkSize*file.wpending >= 1024*1024 {
|
||||
// Hold on.. we got a MB pending.
|
||||
file.c.Wait()
|
||||
if file.err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
file.wpending++
|
||||
|
||||
debugf("GridFile %p: inserting chunk %d with %d bytes", file, n, len(data))
|
||||
|
||||
// We may not own the memory of data, so rather than
|
||||
// simply copying it, we'll marshal the document ahead of time.
|
||||
data, err := bson.Marshal(gfsChunk{bson.NewObjectId(), file.doc.Id, n, data})
|
||||
if err != nil {
|
||||
file.err = err
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := file.gfs.Chunks.Insert(bson.Raw{Data: data})
|
||||
file.m.Lock()
|
||||
file.wpending--
|
||||
if err != nil && file.err == nil {
|
||||
file.err = err
|
||||
}
|
||||
file.c.Broadcast()
|
||||
file.m.Unlock()
|
||||
}()
|
||||
}
|
||||
|
||||
// Seek sets the offset for the next Read or Write on file to
|
||||
// offset, interpreted according to whence: 0 means relative to
|
||||
// the origin of the file, 1 means relative to the current offset,
|
||||
// and 2 means relative to the end. It returns the new offset and
|
||||
// an error, if any.
|
||||
func (file *GridFile) Seek(offset int64, whence int) (pos int64, err error) {
|
||||
file.m.Lock()
|
||||
debugf("GridFile %p: seeking for %s (whence=%d)", file, offset, whence)
|
||||
defer file.m.Unlock()
|
||||
switch whence {
|
||||
case os.SEEK_SET:
|
||||
case os.SEEK_CUR:
|
||||
offset += file.offset
|
||||
case os.SEEK_END:
|
||||
offset += file.doc.Length
|
||||
default:
|
||||
panic("unsupported whence value")
|
||||
}
|
||||
if offset > file.doc.Length {
|
||||
return file.offset, errors.New("seek past end of file")
|
||||
}
|
||||
if offset == file.doc.Length {
|
||||
// If we're seeking to the end of the file,
|
||||
// no need to read anything. This enables
|
||||
// a client to find the size of the file using only the
|
||||
// io.ReadSeeker interface with low overhead.
|
||||
file.offset = offset
|
||||
return file.offset, nil
|
||||
}
|
||||
chunk := int(offset / int64(file.doc.ChunkSize))
|
||||
if chunk+1 == file.chunk && offset >= file.offset {
|
||||
file.rbuf = file.rbuf[int(offset-file.offset):]
|
||||
file.offset = offset
|
||||
return file.offset, nil
|
||||
}
|
||||
file.offset = offset
|
||||
file.chunk = chunk
|
||||
file.rbuf = nil
|
||||
file.rbuf, err = file.getChunk()
|
||||
if err == nil {
|
||||
file.rbuf = file.rbuf[int(file.offset-int64(chunk)*int64(file.doc.ChunkSize)):]
|
||||
}
|
||||
return file.offset, err
|
||||
}
|
||||
|
||||
// Read reads into b the next available data from the file and
|
||||
// returns the number of bytes written and an error in case
|
||||
// something wrong happened. At the end of the file, n will
|
||||
// be zero and err will be set to io.EOF.
|
||||
//
|
||||
// The parameters and behavior of this function turn the file
|
||||
// into an io.Reader.
|
||||
func (file *GridFile) Read(b []byte) (n int, err error) {
|
||||
file.assertMode(gfsReading)
|
||||
file.m.Lock()
|
||||
debugf("GridFile %p: reading at offset %d into buffer of length %d", file, file.offset, len(b))
|
||||
defer file.m.Unlock()
|
||||
if file.offset == file.doc.Length {
|
||||
return 0, io.EOF
|
||||
}
|
||||
for err == nil {
|
||||
i := copy(b, file.rbuf)
|
||||
n += i
|
||||
file.offset += int64(i)
|
||||
file.rbuf = file.rbuf[i:]
|
||||
if i == len(b) || file.offset == file.doc.Length {
|
||||
break
|
||||
}
|
||||
b = b[i:]
|
||||
file.rbuf, err = file.getChunk()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (file *GridFile) getChunk() (data []byte, err error) {
|
||||
cache := file.rcache
|
||||
file.rcache = nil
|
||||
if cache != nil && cache.n == file.chunk {
|
||||
debugf("GridFile %p: Getting chunk %d from cache", file, file.chunk)
|
||||
cache.wait.Lock()
|
||||
data, err = cache.data, cache.err
|
||||
} else {
|
||||
debugf("GridFile %p: Fetching chunk %d", file, file.chunk)
|
||||
var doc gfsChunk
|
||||
err = file.gfs.Chunks.Find(bson.D{{Name: "files_id", Value: file.doc.Id}, {Name: "n", Value: file.chunk}}).One(&doc)
|
||||
data = doc.Data
|
||||
}
|
||||
file.chunk++
|
||||
if int64(file.chunk)*int64(file.doc.ChunkSize) < file.doc.Length {
|
||||
// Read the next one in background.
|
||||
cache = &gfsCachedChunk{n: file.chunk}
|
||||
cache.wait.Lock()
|
||||
debugf("GridFile %p: Scheduling chunk %d for background caching", file, file.chunk)
|
||||
// Clone the session to avoid having it closed in between.
|
||||
chunks := file.gfs.Chunks
|
||||
session := chunks.Database.Session.Clone()
|
||||
go func(id interface{}, n int) {
|
||||
defer session.Close()
|
||||
chunks = chunks.With(session)
|
||||
var doc gfsChunk
|
||||
cache.err = chunks.Find(bson.D{{Name: "files_id", Value: id}, {Name: "n", Value: n}}).One(&doc)
|
||||
cache.data = doc.Data
|
||||
cache.wait.Unlock()
|
||||
}(file.doc.Id, file.chunk)
|
||||
file.rcache = cache
|
||||
}
|
||||
debugf("Returning err: %#v", err)
|
||||
return
|
||||
}
|
||||
-1685
File diff suppressed because it is too large
Load Diff
-1260
File diff suppressed because it is too large
Load Diff
-95
@@ -1,95 +0,0 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// Extension holds a set of additional rules to be used when unmarshaling
|
||||
// strict JSON or JSON-like content.
|
||||
type Extension struct {
|
||||
funcs map[string]funcExt
|
||||
consts map[string]interface{}
|
||||
keyed map[string]func([]byte) (interface{}, error)
|
||||
encode map[reflect.Type]func(v interface{}) ([]byte, error)
|
||||
|
||||
unquotedKeys bool
|
||||
trailingCommas bool
|
||||
}
|
||||
|
||||
type funcExt struct {
|
||||
key string
|
||||
args []string
|
||||
}
|
||||
|
||||
// Extend changes the decoder behavior to consider the provided extension.
|
||||
func (dec *Decoder) Extend(ext *Extension) { dec.d.ext = *ext }
|
||||
|
||||
// Extend changes the encoder behavior to consider the provided extension.
|
||||
func (enc *Encoder) Extend(ext *Extension) { enc.ext = *ext }
|
||||
|
||||
// Extend includes in e the extensions defined in ext.
|
||||
func (e *Extension) Extend(ext *Extension) {
|
||||
for name, fext := range ext.funcs {
|
||||
e.DecodeFunc(name, fext.key, fext.args...)
|
||||
}
|
||||
for name, value := range ext.consts {
|
||||
e.DecodeConst(name, value)
|
||||
}
|
||||
for key, decode := range ext.keyed {
|
||||
e.DecodeKeyed(key, decode)
|
||||
}
|
||||
for typ, encode := range ext.encode {
|
||||
if e.encode == nil {
|
||||
e.encode = make(map[reflect.Type]func(v interface{}) ([]byte, error))
|
||||
}
|
||||
e.encode[typ] = encode
|
||||
}
|
||||
}
|
||||
|
||||
// DecodeFunc defines a function call that may be observed inside JSON content.
|
||||
// A function with the provided name will be unmarshaled as the document
|
||||
// {key: {args[0]: ..., args[N]: ...}}.
|
||||
func (e *Extension) DecodeFunc(name string, key string, args ...string) {
|
||||
if e.funcs == nil {
|
||||
e.funcs = make(map[string]funcExt)
|
||||
}
|
||||
e.funcs[name] = funcExt{key, args}
|
||||
}
|
||||
|
||||
// DecodeConst defines a constant name that may be observed inside JSON content
|
||||
// and will be decoded with the provided value.
|
||||
func (e *Extension) DecodeConst(name string, value interface{}) {
|
||||
if e.consts == nil {
|
||||
e.consts = make(map[string]interface{})
|
||||
}
|
||||
e.consts[name] = value
|
||||
}
|
||||
|
||||
// DecodeKeyed defines a key that when observed as the first element inside a
|
||||
// JSON document triggers the decoding of that document via the provided
|
||||
// decode function.
|
||||
func (e *Extension) DecodeKeyed(key string, decode func(data []byte) (interface{}, error)) {
|
||||
if e.keyed == nil {
|
||||
e.keyed = make(map[string]func([]byte) (interface{}, error))
|
||||
}
|
||||
e.keyed[key] = decode
|
||||
}
|
||||
|
||||
// DecodeUnquotedKeys defines whether to accept map keys that are unquoted strings.
|
||||
func (e *Extension) DecodeUnquotedKeys(accept bool) {
|
||||
e.unquotedKeys = accept
|
||||
}
|
||||
|
||||
// DecodeTrailingCommas defines whether to accept trailing commas in maps and arrays.
|
||||
func (e *Extension) DecodeTrailingCommas(accept bool) {
|
||||
e.trailingCommas = accept
|
||||
}
|
||||
|
||||
// EncodeType registers a function to encode values with the same type of the
|
||||
// provided sample.
|
||||
func (e *Extension) EncodeType(sample interface{}, encode func(v interface{}) ([]byte, error)) {
|
||||
if e.encode == nil {
|
||||
e.encode = make(map[reflect.Type]func(v interface{}) ([]byte, error))
|
||||
}
|
||||
e.encode[reflect.TypeOf(sample)] = encode
|
||||
}
|
||||
-143
@@ -1,143 +0,0 @@
|
||||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package json
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
caseMask = ^byte(0x20) // Mask to ignore case in ASCII.
|
||||
kelvin = '\u212a'
|
||||
smallLongEss = '\u017f'
|
||||
)
|
||||
|
||||
// foldFunc returns one of four different case folding equivalence
|
||||
// functions, from most general (and slow) to fastest:
|
||||
//
|
||||
// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8
|
||||
// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S')
|
||||
// 3) asciiEqualFold, no special, but includes non-letters (including _)
|
||||
// 4) simpleLetterEqualFold, no specials, no non-letters.
|
||||
//
|
||||
// The letters S and K are special because they map to 3 runes, not just 2:
|
||||
// * S maps to s and to U+017F 'ſ' Latin small letter long s
|
||||
// * k maps to K and to U+212A 'K' Kelvin sign
|
||||
// See https://play.golang.org/p/tTxjOc0OGo
|
||||
//
|
||||
// The returned function is specialized for matching against s and
|
||||
// should only be given s. It's not curried for performance reasons.
|
||||
func foldFunc(s []byte) func(s, t []byte) bool {
|
||||
nonLetter := false
|
||||
special := false // special letter
|
||||
for _, b := range s {
|
||||
if b >= utf8.RuneSelf {
|
||||
return bytes.EqualFold
|
||||
}
|
||||
upper := b & caseMask
|
||||
if upper < 'A' || upper > 'Z' {
|
||||
nonLetter = true
|
||||
} else if upper == 'K' || upper == 'S' {
|
||||
// See above for why these letters are special.
|
||||
special = true
|
||||
}
|
||||
}
|
||||
if special {
|
||||
return equalFoldRight
|
||||
}
|
||||
if nonLetter {
|
||||
return asciiEqualFold
|
||||
}
|
||||
return simpleLetterEqualFold
|
||||
}
|
||||
|
||||
// equalFoldRight is a specialization of bytes.EqualFold when s is
|
||||
// known to be all ASCII (including punctuation), but contains an 's',
|
||||
// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t.
|
||||
// See comments on foldFunc.
|
||||
func equalFoldRight(s, t []byte) bool {
|
||||
for _, sb := range s {
|
||||
if len(t) == 0 {
|
||||
return false
|
||||
}
|
||||
tb := t[0]
|
||||
if tb < utf8.RuneSelf {
|
||||
if sb != tb {
|
||||
sbUpper := sb & caseMask
|
||||
if 'A' <= sbUpper && sbUpper <= 'Z' {
|
||||
if sbUpper != tb&caseMask {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
t = t[1:]
|
||||
continue
|
||||
}
|
||||
// sb is ASCII and t is not. t must be either kelvin
|
||||
// sign or long s; sb must be s, S, k, or K.
|
||||
tr, size := utf8.DecodeRune(t)
|
||||
switch sb {
|
||||
case 's', 'S':
|
||||
if tr != smallLongEss {
|
||||
return false
|
||||
}
|
||||
case 'k', 'K':
|
||||
if tr != kelvin {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
t = t[size:]
|
||||
|
||||
}
|
||||
if len(t) > 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// asciiEqualFold is a specialization of bytes.EqualFold for use when
|
||||
// s is all ASCII (but may contain non-letters) and contains no
|
||||
// special-folding letters.
|
||||
// See comments on foldFunc.
|
||||
func asciiEqualFold(s, t []byte) bool {
|
||||
if len(s) != len(t) {
|
||||
return false
|
||||
}
|
||||
for i, sb := range s {
|
||||
tb := t[i]
|
||||
if sb == tb {
|
||||
continue
|
||||
}
|
||||
if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') {
|
||||
if sb&caseMask != tb&caseMask {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// simpleLetterEqualFold is a specialization of bytes.EqualFold for
|
||||
// use when s is all ASCII letters (no underscores, etc) and also
|
||||
// doesn't contain 'k', 'K', 's', or 'S'.
|
||||
// See comments on foldFunc.
|
||||
func simpleLetterEqualFold(s, t []byte) bool {
|
||||
if len(s) != len(t) {
|
||||
return false
|
||||
}
|
||||
for i, b := range s {
|
||||
if b&caseMask != t[i]&caseMask {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
-141
@@ -1,141 +0,0 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package json
|
||||
|
||||
import "bytes"
|
||||
|
||||
// Compact appends to dst the JSON-encoded src with
|
||||
// insignificant space characters elided.
|
||||
func Compact(dst *bytes.Buffer, src []byte) error {
|
||||
return compact(dst, src, false)
|
||||
}
|
||||
|
||||
func compact(dst *bytes.Buffer, src []byte, escape bool) error {
|
||||
origLen := dst.Len()
|
||||
var scan scanner
|
||||
scan.reset()
|
||||
start := 0
|
||||
for i, c := range src {
|
||||
if escape && (c == '<' || c == '>' || c == '&') {
|
||||
if start < i {
|
||||
dst.Write(src[start:i])
|
||||
}
|
||||
dst.WriteString(`\u00`)
|
||||
dst.WriteByte(hex[c>>4])
|
||||
dst.WriteByte(hex[c&0xF])
|
||||
start = i + 1
|
||||
}
|
||||
// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
|
||||
if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
|
||||
if start < i {
|
||||
dst.Write(src[start:i])
|
||||
}
|
||||
dst.WriteString(`\u202`)
|
||||
dst.WriteByte(hex[src[i+2]&0xF])
|
||||
start = i + 3
|
||||
}
|
||||
v := scan.step(&scan, c)
|
||||
if v >= scanSkipSpace {
|
||||
if v == scanError {
|
||||
break
|
||||
}
|
||||
if start < i {
|
||||
dst.Write(src[start:i])
|
||||
}
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
if scan.eof() == scanError {
|
||||
dst.Truncate(origLen)
|
||||
return scan.err
|
||||
}
|
||||
if start < len(src) {
|
||||
dst.Write(src[start:])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newline(dst *bytes.Buffer, prefix, indent string, depth int) {
|
||||
dst.WriteByte('\n')
|
||||
dst.WriteString(prefix)
|
||||
for i := 0; i < depth; i++ {
|
||||
dst.WriteString(indent)
|
||||
}
|
||||
}
|
||||
|
||||
// Indent appends to dst an indented form of the JSON-encoded src.
|
||||
// Each element in a JSON object or array begins on a new,
|
||||
// indented line beginning with prefix followed by one or more
|
||||
// copies of indent according to the indentation nesting.
|
||||
// The data appended to dst does not begin with the prefix nor
|
||||
// any indentation, to make it easier to embed inside other formatted JSON data.
|
||||
// Although leading space characters (space, tab, carriage return, newline)
|
||||
// at the beginning of src are dropped, trailing space characters
|
||||
// at the end of src are preserved and copied to dst.
|
||||
// For example, if src has no trailing spaces, neither will dst;
|
||||
// if src ends in a trailing newline, so will dst.
|
||||
func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
|
||||
origLen := dst.Len()
|
||||
var scan scanner
|
||||
scan.reset()
|
||||
needIndent := false
|
||||
depth := 0
|
||||
for _, c := range src {
|
||||
scan.bytes++
|
||||
v := scan.step(&scan, c)
|
||||
if v == scanSkipSpace {
|
||||
continue
|
||||
}
|
||||
if v == scanError {
|
||||
break
|
||||
}
|
||||
if needIndent && v != scanEndObject && v != scanEndArray {
|
||||
needIndent = false
|
||||
depth++
|
||||
newline(dst, prefix, indent, depth)
|
||||
}
|
||||
|
||||
// Emit semantically uninteresting bytes
|
||||
// (in particular, punctuation in strings) unmodified.
|
||||
if v == scanContinue {
|
||||
dst.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
|
||||
// Add spacing around real punctuation.
|
||||
switch c {
|
||||
case '{', '[':
|
||||
// delay indent so that empty object and array are formatted as {} and [].
|
||||
needIndent = true
|
||||
dst.WriteByte(c)
|
||||
|
||||
case ',':
|
||||
dst.WriteByte(c)
|
||||
newline(dst, prefix, indent, depth)
|
||||
|
||||
case ':':
|
||||
dst.WriteByte(c)
|
||||
dst.WriteByte(' ')
|
||||
|
||||
case '}', ']':
|
||||
if needIndent {
|
||||
// suppress indent in empty object/array
|
||||
needIndent = false
|
||||
} else {
|
||||
depth--
|
||||
newline(dst, prefix, indent, depth)
|
||||
}
|
||||
dst.WriteByte(c)
|
||||
|
||||
default:
|
||||
dst.WriteByte(c)
|
||||
}
|
||||
}
|
||||
if scan.eof() == scanError {
|
||||
dst.Truncate(origLen)
|
||||
return scan.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-697
@@ -1,697 +0,0 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package json
|
||||
|
||||
// JSON value parser state machine.
|
||||
// Just about at the limit of what is reasonable to write by hand.
|
||||
// Some parts are a bit tedious, but overall it nicely factors out the
|
||||
// otherwise common code from the multiple scanning functions
|
||||
// in this package (Compact, Indent, checkValid, nextValue, etc).
|
||||
//
|
||||
// This file starts with two simple examples using the scanner
|
||||
// before diving into the scanner itself.
|
||||
|
||||
import "strconv"
|
||||
|
||||
// checkValid verifies that data is valid JSON-encoded data.
|
||||
// scan is passed in for use by checkValid to avoid an allocation.
|
||||
func checkValid(data []byte, scan *scanner) error {
|
||||
scan.reset()
|
||||
for _, c := range data {
|
||||
scan.bytes++
|
||||
if scan.step(scan, c) == scanError {
|
||||
return scan.err
|
||||
}
|
||||
}
|
||||
if scan.eof() == scanError {
|
||||
return scan.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextValue splits data after the next whole JSON value,
|
||||
// returning that value and the bytes that follow it as separate slices.
|
||||
// scan is passed in for use by nextValue to avoid an allocation.
|
||||
func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) {
|
||||
scan.reset()
|
||||
for i, c := range data {
|
||||
v := scan.step(scan, c)
|
||||
if v >= scanEndObject {
|
||||
switch v {
|
||||
// probe the scanner with a space to determine whether we will
|
||||
// get scanEnd on the next character. Otherwise, if the next character
|
||||
// is not a space, scanEndTop allocates a needless error.
|
||||
case scanEndObject, scanEndArray, scanEndParams:
|
||||
if scan.step(scan, ' ') == scanEnd {
|
||||
return data[:i+1], data[i+1:], nil
|
||||
}
|
||||
case scanError:
|
||||
return nil, nil, scan.err
|
||||
case scanEnd:
|
||||
return data[:i], data[i:], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if scan.eof() == scanError {
|
||||
return nil, nil, scan.err
|
||||
}
|
||||
return data, nil, nil
|
||||
}
|
||||
|
||||
// A SyntaxError is a description of a JSON syntax error.
|
||||
type SyntaxError struct {
|
||||
msg string // description of error
|
||||
Offset int64 // error occurred after reading Offset bytes
|
||||
}
|
||||
|
||||
func (e *SyntaxError) Error() string { return e.msg }
|
||||
|
||||
// A scanner is a JSON scanning state machine.
|
||||
// Callers call scan.reset() and then pass bytes in one at a time
|
||||
// by calling scan.step(&scan, c) for each byte.
|
||||
// The return value, referred to as an opcode, tells the
|
||||
// caller about significant parsing events like beginning
|
||||
// and ending literals, objects, and arrays, so that the
|
||||
// caller can follow along if it wishes.
|
||||
// The return value scanEnd indicates that a single top-level
|
||||
// JSON value has been completed, *before* the byte that
|
||||
// just got passed in. (The indication must be delayed in order
|
||||
// to recognize the end of numbers: is 123 a whole value or
|
||||
// the beginning of 12345e+6?).
|
||||
type scanner struct {
|
||||
// The step is a func to be called to execute the next transition.
|
||||
// Also tried using an integer constant and a single func
|
||||
// with a switch, but using the func directly was 10% faster
|
||||
// on a 64-bit Mac Mini, and it's nicer to read.
|
||||
step func(*scanner, byte) int
|
||||
|
||||
// Reached end of top-level value.
|
||||
endTop bool
|
||||
|
||||
// Stack of what we're in the middle of - array values, object keys, object values.
|
||||
parseState []int
|
||||
|
||||
// Error that happened, if any.
|
||||
err error
|
||||
|
||||
// 1-byte redo (see undo method)
|
||||
redo bool
|
||||
redoCode int
|
||||
redoState func(*scanner, byte) int
|
||||
|
||||
// total bytes consumed, updated by decoder.Decode
|
||||
bytes int64
|
||||
}
|
||||
|
||||
// These values are returned by the state transition functions
|
||||
// assigned to scanner.state and the method scanner.eof.
|
||||
// They give details about the current state of the scan that
|
||||
// callers might be interested to know about.
|
||||
// It is okay to ignore the return value of any particular
|
||||
// call to scanner.state: if one call returns scanError,
|
||||
// every subsequent call will return scanError too.
|
||||
const (
|
||||
// Continue.
|
||||
scanContinue = iota // uninteresting byte
|
||||
scanBeginLiteral // end implied by next result != scanContinue
|
||||
scanBeginObject // begin object
|
||||
scanObjectKey // just finished object key (string)
|
||||
scanObjectValue // just finished non-last object value
|
||||
scanEndObject // end object (implies scanObjectValue if possible)
|
||||
scanBeginArray // begin array
|
||||
scanArrayValue // just finished array value
|
||||
scanEndArray // end array (implies scanArrayValue if possible)
|
||||
scanBeginName // begin function call
|
||||
scanParam // begin function argument
|
||||
scanEndParams // end function call
|
||||
scanSkipSpace // space byte; can skip; known to be last "continue" result
|
||||
|
||||
// Stop.
|
||||
scanEnd // top-level value ended *before* this byte; known to be first "stop" result
|
||||
scanError // hit an error, scanner.err.
|
||||
)
|
||||
|
||||
// These values are stored in the parseState stack.
|
||||
// They give the current state of a composite value
|
||||
// being scanned. If the parser is inside a nested value
|
||||
// the parseState describes the nested state, outermost at entry 0.
|
||||
const (
|
||||
parseObjectKey = iota // parsing object key (before colon)
|
||||
parseObjectValue // parsing object value (after colon)
|
||||
parseArrayValue // parsing array value
|
||||
parseName // parsing unquoted name
|
||||
parseParam // parsing function argument value
|
||||
)
|
||||
|
||||
// reset prepares the scanner for use.
|
||||
// It must be called before calling s.step.
|
||||
func (s *scanner) reset() {
|
||||
s.step = stateBeginValue
|
||||
s.parseState = s.parseState[0:0]
|
||||
s.err = nil
|
||||
s.redo = false
|
||||
s.endTop = false
|
||||
}
|
||||
|
||||
// eof tells the scanner that the end of input has been reached.
|
||||
// It returns a scan status just as s.step does.
|
||||
func (s *scanner) eof() int {
|
||||
if s.err != nil {
|
||||
return scanError
|
||||
}
|
||||
if s.endTop {
|
||||
return scanEnd
|
||||
}
|
||||
s.step(s, ' ')
|
||||
if s.endTop {
|
||||
return scanEnd
|
||||
}
|
||||
if s.err == nil {
|
||||
s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
|
||||
}
|
||||
return scanError
|
||||
}
|
||||
|
||||
// pushParseState pushes a new parse state p onto the parse stack.
|
||||
func (s *scanner) pushParseState(p int) {
|
||||
s.parseState = append(s.parseState, p)
|
||||
}
|
||||
|
||||
// popParseState pops a parse state (already obtained) off the stack
|
||||
// and updates s.step accordingly.
|
||||
func (s *scanner) popParseState() {
|
||||
n := len(s.parseState) - 1
|
||||
s.parseState = s.parseState[0:n]
|
||||
s.redo = false
|
||||
if n == 0 {
|
||||
s.step = stateEndTop
|
||||
s.endTop = true
|
||||
} else {
|
||||
s.step = stateEndValue
|
||||
}
|
||||
}
|
||||
|
||||
func isSpace(c byte) bool {
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n'
|
||||
}
|
||||
|
||||
// stateBeginValueOrEmpty is the state after reading `[`.
|
||||
func stateBeginValueOrEmpty(s *scanner, c byte) int {
|
||||
if c <= ' ' && isSpace(c) {
|
||||
return scanSkipSpace
|
||||
}
|
||||
if c == ']' {
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
return stateBeginValue(s, c)
|
||||
}
|
||||
|
||||
// stateBeginValue is the state at the beginning of the input.
|
||||
func stateBeginValue(s *scanner, c byte) int {
|
||||
if c <= ' ' && isSpace(c) {
|
||||
return scanSkipSpace
|
||||
}
|
||||
switch c {
|
||||
case '{':
|
||||
s.step = stateBeginStringOrEmpty
|
||||
s.pushParseState(parseObjectKey)
|
||||
return scanBeginObject
|
||||
case '[':
|
||||
s.step = stateBeginValueOrEmpty
|
||||
s.pushParseState(parseArrayValue)
|
||||
return scanBeginArray
|
||||
case '"':
|
||||
s.step = stateInString
|
||||
return scanBeginLiteral
|
||||
case '-':
|
||||
s.step = stateNeg
|
||||
return scanBeginLiteral
|
||||
case '0': // beginning of 0.123
|
||||
s.step = state0
|
||||
return scanBeginLiteral
|
||||
case 'n':
|
||||
s.step = stateNew0
|
||||
return scanBeginName
|
||||
}
|
||||
if '1' <= c && c <= '9' { // beginning of 1234.5
|
||||
s.step = state1
|
||||
return scanBeginLiteral
|
||||
}
|
||||
if isName(c) {
|
||||
s.step = stateName
|
||||
return scanBeginName
|
||||
}
|
||||
return s.error(c, "looking for beginning of value")
|
||||
}
|
||||
|
||||
func isName(c byte) bool {
|
||||
return c == '$' || c == '_' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9'
|
||||
}
|
||||
|
||||
// stateBeginStringOrEmpty is the state after reading `{`.
|
||||
func stateBeginStringOrEmpty(s *scanner, c byte) int {
|
||||
if c <= ' ' && isSpace(c) {
|
||||
return scanSkipSpace
|
||||
}
|
||||
if c == '}' {
|
||||
n := len(s.parseState)
|
||||
s.parseState[n-1] = parseObjectValue
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
return stateBeginString(s, c)
|
||||
}
|
||||
|
||||
// stateBeginString is the state after reading `{"key": value,`.
|
||||
func stateBeginString(s *scanner, c byte) int {
|
||||
if c <= ' ' && isSpace(c) {
|
||||
return scanSkipSpace
|
||||
}
|
||||
if c == '"' {
|
||||
s.step = stateInString
|
||||
return scanBeginLiteral
|
||||
}
|
||||
if isName(c) {
|
||||
s.step = stateName
|
||||
return scanBeginName
|
||||
}
|
||||
return s.error(c, "looking for beginning of object key string")
|
||||
}
|
||||
|
||||
// stateEndValue is the state after completing a value,
|
||||
// such as after reading `{}` or `true` or `["x"`.
|
||||
func stateEndValue(s *scanner, c byte) int {
|
||||
n := len(s.parseState)
|
||||
if n == 0 {
|
||||
// Completed top-level before the current byte.
|
||||
s.step = stateEndTop
|
||||
s.endTop = true
|
||||
return stateEndTop(s, c)
|
||||
}
|
||||
if c <= ' ' && isSpace(c) {
|
||||
s.step = stateEndValue
|
||||
return scanSkipSpace
|
||||
}
|
||||
ps := s.parseState[n-1]
|
||||
switch ps {
|
||||
case parseObjectKey:
|
||||
if c == ':' {
|
||||
s.parseState[n-1] = parseObjectValue
|
||||
s.step = stateBeginValue
|
||||
return scanObjectKey
|
||||
}
|
||||
return s.error(c, "after object key")
|
||||
case parseObjectValue:
|
||||
if c == ',' {
|
||||
s.parseState[n-1] = parseObjectKey
|
||||
s.step = stateBeginStringOrEmpty
|
||||
return scanObjectValue
|
||||
}
|
||||
if c == '}' {
|
||||
s.popParseState()
|
||||
return scanEndObject
|
||||
}
|
||||
return s.error(c, "after object key:value pair")
|
||||
case parseArrayValue:
|
||||
if c == ',' {
|
||||
s.step = stateBeginValueOrEmpty
|
||||
return scanArrayValue
|
||||
}
|
||||
if c == ']' {
|
||||
s.popParseState()
|
||||
return scanEndArray
|
||||
}
|
||||
return s.error(c, "after array element")
|
||||
case parseParam:
|
||||
if c == ',' {
|
||||
s.step = stateBeginValue
|
||||
return scanParam
|
||||
}
|
||||
if c == ')' {
|
||||
s.popParseState()
|
||||
return scanEndParams
|
||||
}
|
||||
return s.error(c, "after array element")
|
||||
}
|
||||
return s.error(c, "")
|
||||
}
|
||||
|
||||
// stateEndTop is the state after finishing the top-level value,
|
||||
// such as after reading `{}` or `[1,2,3]`.
|
||||
// Only space characters should be seen now.
|
||||
func stateEndTop(s *scanner, c byte) int {
|
||||
if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
|
||||
// Complain about non-space byte on next call.
|
||||
s.error(c, "after top-level value")
|
||||
}
|
||||
return scanEnd
|
||||
}
|
||||
|
||||
// stateInString is the state after reading `"`.
|
||||
func stateInString(s *scanner, c byte) int {
|
||||
if c == '"' {
|
||||
s.step = stateEndValue
|
||||
return scanContinue
|
||||
}
|
||||
if c == '\\' {
|
||||
s.step = stateInStringEsc
|
||||
return scanContinue
|
||||
}
|
||||
if c < 0x20 {
|
||||
return s.error(c, "in string literal")
|
||||
}
|
||||
return scanContinue
|
||||
}
|
||||
|
||||
// stateInStringEsc is the state after reading `"\` during a quoted string.
|
||||
func stateInStringEsc(s *scanner, c byte) int {
|
||||
switch c {
|
||||
case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
|
||||
s.step = stateInString
|
||||
return scanContinue
|
||||
case 'u':
|
||||
s.step = stateInStringEscU
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in string escape code")
|
||||
}
|
||||
|
||||
// stateInStringEscU is the state after reading `"\u` during a quoted string.
|
||||
func stateInStringEscU(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
|
||||
s.step = stateInStringEscU1
|
||||
return scanContinue
|
||||
}
|
||||
// numbers
|
||||
return s.error(c, "in \\u hexadecimal character escape")
|
||||
}
|
||||
|
||||
// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
|
||||
func stateInStringEscU1(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
|
||||
s.step = stateInStringEscU12
|
||||
return scanContinue
|
||||
}
|
||||
// numbers
|
||||
return s.error(c, "in \\u hexadecimal character escape")
|
||||
}
|
||||
|
||||
// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
|
||||
func stateInStringEscU12(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
|
||||
s.step = stateInStringEscU123
|
||||
return scanContinue
|
||||
}
|
||||
// numbers
|
||||
return s.error(c, "in \\u hexadecimal character escape")
|
||||
}
|
||||
|
||||
// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
|
||||
func stateInStringEscU123(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
|
||||
s.step = stateInString
|
||||
return scanContinue
|
||||
}
|
||||
// numbers
|
||||
return s.error(c, "in \\u hexadecimal character escape")
|
||||
}
|
||||
|
||||
// stateNeg is the state after reading `-` during a number.
|
||||
func stateNeg(s *scanner, c byte) int {
|
||||
if c == '0' {
|
||||
s.step = state0
|
||||
return scanContinue
|
||||
}
|
||||
if '1' <= c && c <= '9' {
|
||||
s.step = state1
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in numeric literal")
|
||||
}
|
||||
|
||||
// state1 is the state after reading a non-zero integer during a number,
|
||||
// such as after reading `1` or `100` but not `0`.
|
||||
func state1(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' {
|
||||
s.step = state1
|
||||
return scanContinue
|
||||
}
|
||||
return state0(s, c)
|
||||
}
|
||||
|
||||
// state0 is the state after reading `0` during a number.
|
||||
func state0(s *scanner, c byte) int {
|
||||
if c == '.' {
|
||||
s.step = stateDot
|
||||
return scanContinue
|
||||
}
|
||||
if c == 'e' || c == 'E' {
|
||||
s.step = stateE
|
||||
return scanContinue
|
||||
}
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
|
||||
// stateDot is the state after reading the integer and decimal point in a number,
|
||||
// such as after reading `1.`.
|
||||
func stateDot(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' {
|
||||
s.step = stateDot0
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "after decimal point in numeric literal")
|
||||
}
|
||||
|
||||
// stateDot0 is the state after reading the integer, decimal point, and subsequent
|
||||
// digits of a number, such as after reading `3.14`.
|
||||
func stateDot0(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' {
|
||||
return scanContinue
|
||||
}
|
||||
if c == 'e' || c == 'E' {
|
||||
s.step = stateE
|
||||
return scanContinue
|
||||
}
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
|
||||
// stateE is the state after reading the mantissa and e in a number,
|
||||
// such as after reading `314e` or `0.314e`.
|
||||
func stateE(s *scanner, c byte) int {
|
||||
if c == '+' || c == '-' {
|
||||
s.step = stateESign
|
||||
return scanContinue
|
||||
}
|
||||
return stateESign(s, c)
|
||||
}
|
||||
|
||||
// stateESign is the state after reading the mantissa, e, and sign in a number,
|
||||
// such as after reading `314e-` or `0.314e+`.
|
||||
func stateESign(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' {
|
||||
s.step = stateE0
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in exponent of numeric literal")
|
||||
}
|
||||
|
||||
// stateE0 is the state after reading the mantissa, e, optional sign,
|
||||
// and at least one digit of the exponent in a number,
|
||||
// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
|
||||
func stateE0(s *scanner, c byte) int {
|
||||
if '0' <= c && c <= '9' {
|
||||
return scanContinue
|
||||
}
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
|
||||
// stateNew0 is the state after reading `n`.
|
||||
func stateNew0(s *scanner, c byte) int {
|
||||
if c == 'e' {
|
||||
s.step = stateNew1
|
||||
return scanContinue
|
||||
}
|
||||
s.step = stateName
|
||||
return stateName(s, c)
|
||||
}
|
||||
|
||||
// stateNew1 is the state after reading `ne`.
|
||||
func stateNew1(s *scanner, c byte) int {
|
||||
if c == 'w' {
|
||||
s.step = stateNew2
|
||||
return scanContinue
|
||||
}
|
||||
s.step = stateName
|
||||
return stateName(s, c)
|
||||
}
|
||||
|
||||
// stateNew2 is the state after reading `new`.
|
||||
func stateNew2(s *scanner, c byte) int {
|
||||
s.step = stateName
|
||||
if c == ' ' {
|
||||
return scanContinue
|
||||
}
|
||||
return stateName(s, c)
|
||||
}
|
||||
|
||||
// stateName is the state while reading an unquoted function name.
|
||||
func stateName(s *scanner, c byte) int {
|
||||
if isName(c) {
|
||||
return scanContinue
|
||||
}
|
||||
if c == '(' {
|
||||
s.step = stateParamOrEmpty
|
||||
s.pushParseState(parseParam)
|
||||
return scanParam
|
||||
}
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
|
||||
// stateParamOrEmpty is the state after reading `(`.
|
||||
func stateParamOrEmpty(s *scanner, c byte) int {
|
||||
if c <= ' ' && isSpace(c) {
|
||||
return scanSkipSpace
|
||||
}
|
||||
if c == ')' {
|
||||
return stateEndValue(s, c)
|
||||
}
|
||||
return stateBeginValue(s, c)
|
||||
}
|
||||
|
||||
// stateT is the state after reading `t`.
|
||||
func stateT(s *scanner, c byte) int {
|
||||
if c == 'r' {
|
||||
s.step = stateTr
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal true (expecting 'r')")
|
||||
}
|
||||
|
||||
// stateTr is the state after reading `tr`.
|
||||
func stateTr(s *scanner, c byte) int {
|
||||
if c == 'u' {
|
||||
s.step = stateTru
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal true (expecting 'u')")
|
||||
}
|
||||
|
||||
// stateTru is the state after reading `tru`.
|
||||
func stateTru(s *scanner, c byte) int {
|
||||
if c == 'e' {
|
||||
s.step = stateEndValue
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal true (expecting 'e')")
|
||||
}
|
||||
|
||||
// stateF is the state after reading `f`.
|
||||
func stateF(s *scanner, c byte) int {
|
||||
if c == 'a' {
|
||||
s.step = stateFa
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal false (expecting 'a')")
|
||||
}
|
||||
|
||||
// stateFa is the state after reading `fa`.
|
||||
func stateFa(s *scanner, c byte) int {
|
||||
if c == 'l' {
|
||||
s.step = stateFal
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal false (expecting 'l')")
|
||||
}
|
||||
|
||||
// stateFal is the state after reading `fal`.
|
||||
func stateFal(s *scanner, c byte) int {
|
||||
if c == 's' {
|
||||
s.step = stateFals
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal false (expecting 's')")
|
||||
}
|
||||
|
||||
// stateFals is the state after reading `fals`.
|
||||
func stateFals(s *scanner, c byte) int {
|
||||
if c == 'e' {
|
||||
s.step = stateEndValue
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal false (expecting 'e')")
|
||||
}
|
||||
|
||||
// stateN is the state after reading `n`.
|
||||
func stateN(s *scanner, c byte) int {
|
||||
if c == 'u' {
|
||||
s.step = stateNu
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal null (expecting 'u')")
|
||||
}
|
||||
|
||||
// stateNu is the state after reading `nu`.
|
||||
func stateNu(s *scanner, c byte) int {
|
||||
if c == 'l' {
|
||||
s.step = stateNul
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal null (expecting 'l')")
|
||||
}
|
||||
|
||||
// stateNul is the state after reading `nul`.
|
||||
func stateNul(s *scanner, c byte) int {
|
||||
if c == 'l' {
|
||||
s.step = stateEndValue
|
||||
return scanContinue
|
||||
}
|
||||
return s.error(c, "in literal null (expecting 'l')")
|
||||
}
|
||||
|
||||
// stateError is the state after reaching a syntax error,
|
||||
// such as after reading `[1}` or `5.1.2`.
|
||||
func stateError(s *scanner, c byte) int {
|
||||
return scanError
|
||||
}
|
||||
|
||||
// error records an error and switches to the error state.
|
||||
func (s *scanner) error(c byte, context string) int {
|
||||
s.step = stateError
|
||||
s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
|
||||
return scanError
|
||||
}
|
||||
|
||||
// quoteChar formats c as a quoted character literal
|
||||
func quoteChar(c byte) string {
|
||||
// special cases - different from quoted strings
|
||||
if c == '\'' {
|
||||
return `'\''`
|
||||
}
|
||||
if c == '"' {
|
||||
return `'"'`
|
||||
}
|
||||
|
||||
// use quoted string with different quotation marks
|
||||
s := strconv.Quote(string(c))
|
||||
return "'" + s[1:len(s)-1] + "'"
|
||||
}
|
||||
|
||||
// undo causes the scanner to return scanCode from the next state transition.
|
||||
// This gives callers a simple 1-byte undo mechanism.
|
||||
func (s *scanner) undo(scanCode int) {
|
||||
if s.redo {
|
||||
panic("json: invalid use of scanner")
|
||||
}
|
||||
s.redoCode = scanCode
|
||||
s.redoState = s.step
|
||||
s.step = stateRedo
|
||||
s.redo = true
|
||||
}
|
||||
|
||||
// stateRedo helps implement the scanner's 1-byte undo.
|
||||
func stateRedo(s *scanner, c byte) int {
|
||||
s.redo = false
|
||||
s.step = s.redoState
|
||||
return s.redoCode
|
||||
}
|
||||
-510
@@ -1,510 +0,0 @@
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package json
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// A Decoder reads and decodes JSON values from an input stream.
|
||||
type Decoder struct {
|
||||
r io.Reader
|
||||
buf []byte
|
||||
d decodeState
|
||||
scanp int // start of unread data in buf
|
||||
scan scanner
|
||||
err error
|
||||
|
||||
tokenState int
|
||||
tokenStack []int
|
||||
}
|
||||
|
||||
// NewDecoder returns a new decoder that reads from r.
|
||||
//
|
||||
// The decoder introduces its own buffering and may
|
||||
// read data from r beyond the JSON values requested.
|
||||
func NewDecoder(r io.Reader) *Decoder {
|
||||
return &Decoder{r: r}
|
||||
}
|
||||
|
||||
// UseNumber causes the Decoder to unmarshal a number into an interface{} as a
|
||||
// Number instead of as a float64.
|
||||
func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
|
||||
|
||||
// Decode reads the next JSON-encoded value from its
|
||||
// input and stores it in the value pointed to by v.
|
||||
//
|
||||
// See the documentation for Unmarshal for details about
|
||||
// the conversion of JSON into a Go value.
|
||||
func (dec *Decoder) Decode(v interface{}) error {
|
||||
if dec.err != nil {
|
||||
return dec.err
|
||||
}
|
||||
|
||||
if err := dec.tokenPrepareForDecode(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !dec.tokenValueAllowed() {
|
||||
return &SyntaxError{msg: "not at beginning of value"}
|
||||
}
|
||||
|
||||
// Read whole value into buffer.
|
||||
n, err := dec.readValue()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
|
||||
dec.scanp += n
|
||||
|
||||
// Don't save err from unmarshal into dec.err:
|
||||
// the connection is still usable since we read a complete JSON
|
||||
// object from it before the error happened.
|
||||
err = dec.d.unmarshal(v)
|
||||
|
||||
// fixup token streaming state
|
||||
dec.tokenValueEnd()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Buffered returns a reader of the data remaining in the Decoder's
|
||||
// buffer. The reader is valid until the next call to Decode.
|
||||
func (dec *Decoder) Buffered() io.Reader {
|
||||
return bytes.NewReader(dec.buf[dec.scanp:])
|
||||
}
|
||||
|
||||
// readValue reads a JSON value into dec.buf.
|
||||
// It returns the length of the encoding.
|
||||
func (dec *Decoder) readValue() (int, error) {
|
||||
dec.scan.reset()
|
||||
|
||||
scanp := dec.scanp
|
||||
var err error
|
||||
Input:
|
||||
for {
|
||||
// Look in the buffer for a new value.
|
||||
for i, c := range dec.buf[scanp:] {
|
||||
dec.scan.bytes++
|
||||
v := dec.scan.step(&dec.scan, c)
|
||||
if v == scanEnd {
|
||||
scanp += i
|
||||
break Input
|
||||
}
|
||||
// scanEnd is delayed one byte.
|
||||
// We might block trying to get that byte from src,
|
||||
// so instead invent a space byte.
|
||||
if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd {
|
||||
scanp += i + 1
|
||||
break Input
|
||||
}
|
||||
if v == scanError {
|
||||
dec.err = dec.scan.err
|
||||
return 0, dec.scan.err
|
||||
}
|
||||
}
|
||||
scanp = len(dec.buf)
|
||||
|
||||
// Did the last read have an error?
|
||||
// Delayed until now to allow buffer scan.
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if dec.scan.step(&dec.scan, ' ') == scanEnd {
|
||||
break Input
|
||||
}
|
||||
if nonSpace(dec.buf) {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
}
|
||||
dec.err = err
|
||||
return 0, err
|
||||
}
|
||||
|
||||
n := scanp - dec.scanp
|
||||
err = dec.refill()
|
||||
scanp = dec.scanp + n
|
||||
}
|
||||
return scanp - dec.scanp, nil
|
||||
}
|
||||
|
||||
func (dec *Decoder) refill() error {
|
||||
// Make room to read more into the buffer.
|
||||
// First slide down data already consumed.
|
||||
if dec.scanp > 0 {
|
||||
n := copy(dec.buf, dec.buf[dec.scanp:])
|
||||
dec.buf = dec.buf[:n]
|
||||
dec.scanp = 0
|
||||
}
|
||||
|
||||
// Grow buffer if not large enough.
|
||||
const minRead = 512
|
||||
if cap(dec.buf)-len(dec.buf) < minRead {
|
||||
newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead)
|
||||
copy(newBuf, dec.buf)
|
||||
dec.buf = newBuf
|
||||
}
|
||||
|
||||
// Read. Delay error for next iteration (after scan).
|
||||
n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
|
||||
dec.buf = dec.buf[0 : len(dec.buf)+n]
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func nonSpace(b []byte) bool {
|
||||
for _, c := range b {
|
||||
if !isSpace(c) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// An Encoder writes JSON values to an output stream.
|
||||
type Encoder struct {
|
||||
w io.Writer
|
||||
err error
|
||||
escapeHTML bool
|
||||
|
||||
indentBuf *bytes.Buffer
|
||||
indentPrefix string
|
||||
indentValue string
|
||||
|
||||
ext Extension
|
||||
}
|
||||
|
||||
// NewEncoder returns a new encoder that writes to w.
|
||||
func NewEncoder(w io.Writer) *Encoder {
|
||||
return &Encoder{w: w, escapeHTML: true}
|
||||
}
|
||||
|
||||
// Encode writes the JSON encoding of v to the stream,
|
||||
// followed by a newline character.
|
||||
//
|
||||
// See the documentation for Marshal for details about the
|
||||
// conversion of Go values to JSON.
|
||||
func (enc *Encoder) Encode(v interface{}) error {
|
||||
if enc.err != nil {
|
||||
return enc.err
|
||||
}
|
||||
e := newEncodeState()
|
||||
e.ext = enc.ext
|
||||
err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Terminate each value with a newline.
|
||||
// This makes the output look a little nicer
|
||||
// when debugging, and some kind of space
|
||||
// is required if the encoded value was a number,
|
||||
// so that the reader knows there aren't more
|
||||
// digits coming.
|
||||
e.WriteByte('\n')
|
||||
|
||||
b := e.Bytes()
|
||||
if enc.indentBuf != nil {
|
||||
enc.indentBuf.Reset()
|
||||
err = Indent(enc.indentBuf, b, enc.indentPrefix, enc.indentValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = enc.indentBuf.Bytes()
|
||||
}
|
||||
if _, err = enc.w.Write(b); err != nil {
|
||||
enc.err = err
|
||||
}
|
||||
encodeStatePool.Put(e)
|
||||
return err
|
||||
}
|
||||
|
||||
// Indent sets the encoder to format each encoded value with Indent.
|
||||
func (enc *Encoder) Indent(prefix, indent string) {
|
||||
enc.indentBuf = new(bytes.Buffer)
|
||||
enc.indentPrefix = prefix
|
||||
enc.indentValue = indent
|
||||
}
|
||||
|
||||
// DisableHTMLEscaping causes the encoder not to escape angle brackets
|
||||
// ("<" and ">") or ampersands ("&") in JSON strings.
|
||||
func (enc *Encoder) DisableHTMLEscaping() {
|
||||
enc.escapeHTML = false
|
||||
}
|
||||
|
||||
// RawMessage is a raw encoded JSON value.
|
||||
// It implements Marshaler and Unmarshaler and can
|
||||
// be used to delay JSON decoding or precompute a JSON encoding.
|
||||
type RawMessage []byte
|
||||
|
||||
// MarshalJSON returns *m as the JSON encoding of m.
|
||||
func (m *RawMessage) MarshalJSON() ([]byte, error) {
|
||||
return *m, nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON sets *m to a copy of data.
|
||||
func (m *RawMessage) UnmarshalJSON(data []byte) error {
|
||||
if m == nil {
|
||||
return errors.New("json.RawMessage: UnmarshalJSON on nil pointer")
|
||||
}
|
||||
*m = append((*m)[0:0], data...)
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ Marshaler = (*RawMessage)(nil)
|
||||
var _ Unmarshaler = (*RawMessage)(nil)
|
||||
|
||||
// A Token holds a value of one of these types:
|
||||
//
|
||||
// Delim, for the four JSON delimiters [ ] { }
|
||||
// bool, for JSON booleans
|
||||
// float64, for JSON numbers
|
||||
// Number, for JSON numbers
|
||||
// string, for JSON string literals
|
||||
// nil, for JSON null
|
||||
//
|
||||
type Token interface{}
|
||||
|
||||
const (
|
||||
tokenTopValue = iota
|
||||
tokenArrayStart
|
||||
tokenArrayValue
|
||||
tokenArrayComma
|
||||
tokenObjectStart
|
||||
tokenObjectKey
|
||||
tokenObjectColon
|
||||
tokenObjectValue
|
||||
tokenObjectComma
|
||||
)
|
||||
|
||||
// advance tokenstate from a separator state to a value state
|
||||
func (dec *Decoder) tokenPrepareForDecode() error {
|
||||
// Note: Not calling peek before switch, to avoid
|
||||
// putting peek into the standard Decode path.
|
||||
// peek is only called when using the Token API.
|
||||
switch dec.tokenState {
|
||||
case tokenArrayComma:
|
||||
c, err := dec.peek()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c != ',' {
|
||||
return &SyntaxError{"expected comma after array element", 0}
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenState = tokenArrayValue
|
||||
case tokenObjectColon:
|
||||
c, err := dec.peek()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c != ':' {
|
||||
return &SyntaxError{"expected colon after object key", 0}
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenState = tokenObjectValue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dec *Decoder) tokenValueAllowed() bool {
|
||||
switch dec.tokenState {
|
||||
case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (dec *Decoder) tokenValueEnd() {
|
||||
switch dec.tokenState {
|
||||
case tokenArrayStart, tokenArrayValue:
|
||||
dec.tokenState = tokenArrayComma
|
||||
case tokenObjectValue:
|
||||
dec.tokenState = tokenObjectComma
|
||||
}
|
||||
}
|
||||
|
||||
// A Delim is a JSON array or object delimiter, one of [ ] { or }.
|
||||
type Delim rune
|
||||
|
||||
func (d Delim) String() string {
|
||||
return string(d)
|
||||
}
|
||||
|
||||
// Token returns the next JSON token in the input stream.
|
||||
// At the end of the input stream, Token returns nil, io.EOF.
|
||||
//
|
||||
// Token guarantees that the delimiters [ ] { } it returns are
|
||||
// properly nested and matched: if Token encounters an unexpected
|
||||
// delimiter in the input, it will return an error.
|
||||
//
|
||||
// The input stream consists of basic JSON values—bool, string,
|
||||
// number, and null—along with delimiters [ ] { } of type Delim
|
||||
// to mark the start and end of arrays and objects.
|
||||
// Commas and colons are elided.
|
||||
func (dec *Decoder) Token() (Token, error) {
|
||||
for {
|
||||
c, err := dec.peek()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch c {
|
||||
case '[':
|
||||
if !dec.tokenValueAllowed() {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenStack = append(dec.tokenStack, dec.tokenState)
|
||||
dec.tokenState = tokenArrayStart
|
||||
return Delim('['), nil
|
||||
|
||||
case ']':
|
||||
if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
|
||||
dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
|
||||
dec.tokenValueEnd()
|
||||
return Delim(']'), nil
|
||||
|
||||
case '{':
|
||||
if !dec.tokenValueAllowed() {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenStack = append(dec.tokenStack, dec.tokenState)
|
||||
dec.tokenState = tokenObjectStart
|
||||
return Delim('{'), nil
|
||||
|
||||
case '}':
|
||||
if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
|
||||
dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
|
||||
dec.tokenValueEnd()
|
||||
return Delim('}'), nil
|
||||
|
||||
case ':':
|
||||
if dec.tokenState != tokenObjectColon {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
dec.scanp++
|
||||
dec.tokenState = tokenObjectValue
|
||||
continue
|
||||
|
||||
case ',':
|
||||
if dec.tokenState == tokenArrayComma {
|
||||
dec.scanp++
|
||||
dec.tokenState = tokenArrayValue
|
||||
continue
|
||||
}
|
||||
if dec.tokenState == tokenObjectComma {
|
||||
dec.scanp++
|
||||
dec.tokenState = tokenObjectKey
|
||||
continue
|
||||
}
|
||||
return dec.tokenError(c)
|
||||
|
||||
case '"':
|
||||
if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey {
|
||||
var x string
|
||||
old := dec.tokenState
|
||||
dec.tokenState = tokenTopValue
|
||||
err := dec.Decode(&x)
|
||||
dec.tokenState = old
|
||||
if err != nil {
|
||||
clearOffset(err)
|
||||
return nil, err
|
||||
}
|
||||
dec.tokenState = tokenObjectColon
|
||||
return x, nil
|
||||
}
|
||||
fallthrough
|
||||
|
||||
default:
|
||||
if !dec.tokenValueAllowed() {
|
||||
return dec.tokenError(c)
|
||||
}
|
||||
var x interface{}
|
||||
if err := dec.Decode(&x); err != nil {
|
||||
clearOffset(err)
|
||||
return nil, err
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func clearOffset(err error) {
|
||||
if s, ok := err.(*SyntaxError); ok {
|
||||
s.Offset = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (dec *Decoder) tokenError(c byte) (Token, error) {
|
||||
var context string
|
||||
switch dec.tokenState {
|
||||
case tokenTopValue:
|
||||
context = " looking for beginning of value"
|
||||
case tokenArrayStart, tokenArrayValue, tokenObjectValue:
|
||||
context = " looking for beginning of value"
|
||||
case tokenArrayComma:
|
||||
context = " after array element"
|
||||
case tokenObjectKey:
|
||||
context = " looking for beginning of object key string"
|
||||
case tokenObjectColon:
|
||||
context = " after object key"
|
||||
case tokenObjectComma:
|
||||
context = " after object key:value pair"
|
||||
}
|
||||
return nil, &SyntaxError{"invalid character " + quoteChar(c) + " " + context, 0}
|
||||
}
|
||||
|
||||
// More reports whether there is another element in the
|
||||
// current array or object being parsed.
|
||||
func (dec *Decoder) More() bool {
|
||||
c, err := dec.peek()
|
||||
return err == nil && c != ']' && c != '}'
|
||||
}
|
||||
|
||||
func (dec *Decoder) peek() (byte, error) {
|
||||
var err error
|
||||
for {
|
||||
for i := dec.scanp; i < len(dec.buf); i++ {
|
||||
c := dec.buf[i]
|
||||
if isSpace(c) {
|
||||
continue
|
||||
}
|
||||
dec.scanp = i
|
||||
return c, nil
|
||||
}
|
||||
// buffer has been scanned, now report any error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
err = dec.refill()
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
TODO
|
||||
|
||||
// EncodeToken writes the given JSON token to the stream.
|
||||
// It returns an error if the delimiters [ ] { } are not properly used.
|
||||
//
|
||||
// EncodeToken does not call Flush, because usually it is part of
|
||||
// a larger operation such as Encode, and those will call Flush when finished.
|
||||
// Callers that create an Encoder and then invoke EncodeToken directly,
|
||||
// without using Encode, need to call Flush when finished to ensure that
|
||||
// the JSON is written to the underlying writer.
|
||||
func (e *Encoder) EncodeToken(t Token) error {
|
||||
...
|
||||
}
|
||||
|
||||
*/
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package json
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// tagOptions is the string following a comma in a struct field's "json"
|
||||
// tag, or the empty string. It does not include the leading comma.
|
||||
type tagOptions string
|
||||
|
||||
// parseTag splits a struct field's json tag into its name and
|
||||
// comma-separated options.
|
||||
func parseTag(tag string) (string, tagOptions) {
|
||||
if idx := strings.Index(tag, ","); idx != -1 {
|
||||
return tag[:idx], tagOptions(tag[idx+1:])
|
||||
}
|
||||
return tag, tagOptions("")
|
||||
}
|
||||
|
||||
// Contains reports whether a comma-separated list of options
|
||||
// contains a particular substr flag. substr must be surrounded by a
|
||||
// string boundary or commas.
|
||||
func (o tagOptions) Contains(optionName string) bool {
|
||||
if len(o) == 0 {
|
||||
return false
|
||||
}
|
||||
s := string(o)
|
||||
for s != "" {
|
||||
var next string
|
||||
i := strings.Index(s, ",")
|
||||
if i >= 0 {
|
||||
s, next = s[:i], s[i+1:]
|
||||
}
|
||||
if s == optionName {
|
||||
return true
|
||||
}
|
||||
s = next
|
||||
}
|
||||
return false
|
||||
}
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
// +build !windows
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sasl/sasl.h>
|
||||
|
||||
static int mgo_sasl_simple(void *context, int id, const char **result, unsigned int *len)
|
||||
{
|
||||
if (!result) {
|
||||
return SASL_BADPARAM;
|
||||
}
|
||||
switch (id) {
|
||||
case SASL_CB_USER:
|
||||
*result = (char *)context;
|
||||
break;
|
||||
case SASL_CB_AUTHNAME:
|
||||
*result = (char *)context;
|
||||
break;
|
||||
case SASL_CB_LANGUAGE:
|
||||
*result = NULL;
|
||||
break;
|
||||
default:
|
||||
return SASL_BADPARAM;
|
||||
}
|
||||
if (len) {
|
||||
*len = *result ? strlen(*result) : 0;
|
||||
}
|
||||
return SASL_OK;
|
||||
}
|
||||
|
||||
typedef int (*callback)(void);
|
||||
|
||||
static int mgo_sasl_secret(sasl_conn_t *conn, void *context, int id, sasl_secret_t **result)
|
||||
{
|
||||
if (!conn || !result || id != SASL_CB_PASS) {
|
||||
return SASL_BADPARAM;
|
||||
}
|
||||
*result = (sasl_secret_t *)context;
|
||||
return SASL_OK;
|
||||
}
|
||||
|
||||
sasl_callback_t *mgo_sasl_callbacks(const char *username, const char *password)
|
||||
{
|
||||
sasl_callback_t *cb = malloc(4 * sizeof(sasl_callback_t));
|
||||
int n = 0;
|
||||
|
||||
size_t len = strlen(password);
|
||||
sasl_secret_t *secret = (sasl_secret_t*)malloc(sizeof(sasl_secret_t) + len);
|
||||
if (!secret) {
|
||||
free(cb);
|
||||
return NULL;
|
||||
}
|
||||
strcpy((char *)secret->data, password);
|
||||
secret->len = len;
|
||||
|
||||
cb[n].id = SASL_CB_PASS;
|
||||
cb[n].proc = (callback)&mgo_sasl_secret;
|
||||
cb[n].context = secret;
|
||||
n++;
|
||||
|
||||
cb[n].id = SASL_CB_USER;
|
||||
cb[n].proc = (callback)&mgo_sasl_simple;
|
||||
cb[n].context = (char*)username;
|
||||
n++;
|
||||
|
||||
cb[n].id = SASL_CB_AUTHNAME;
|
||||
cb[n].proc = (callback)&mgo_sasl_simple;
|
||||
cb[n].context = (char*)username;
|
||||
n++;
|
||||
|
||||
cb[n].id = SASL_CB_LIST_END;
|
||||
cb[n].proc = NULL;
|
||||
cb[n].context = NULL;
|
||||
|
||||
return cb;
|
||||
}
|
||||
-142
@@ -1,142 +0,0 @@
|
||||
// Package sasl is an implementation detail of the mgo package.
|
||||
//
|
||||
// This package is not meant to be used by itself.
|
||||
//
|
||||
|
||||
// +build !windows
|
||||
|
||||
package sasl
|
||||
|
||||
// #cgo LDFLAGS: -lsasl2
|
||||
// #cgo CFLAGS: -Wno-deprecated-declarations
|
||||
//
|
||||
// struct sasl_conn {};
|
||||
//
|
||||
// #include <stdlib.h>
|
||||
// #include <sasl/sasl.h>
|
||||
//
|
||||
// sasl_callback_t *mgo_sasl_callbacks(const char *username, const char *password);
|
||||
//
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Stepper interface for saslSession
|
||||
type Stepper interface {
|
||||
Step(serverData []byte) (clientData []byte, done bool, err error)
|
||||
Close()
|
||||
}
|
||||
|
||||
type saslSession struct {
|
||||
conn *C.sasl_conn_t
|
||||
step int
|
||||
mech string
|
||||
|
||||
cstrings []*C.char
|
||||
callbacks *C.sasl_callback_t
|
||||
}
|
||||
|
||||
var initError error
|
||||
var initOnce sync.Once
|
||||
|
||||
func initSASL() {
|
||||
rc := C.sasl_client_init(nil)
|
||||
if rc != C.SASL_OK {
|
||||
initError = saslError(rc, nil, "cannot initialize SASL library")
|
||||
}
|
||||
}
|
||||
|
||||
// New creates a new saslSession
|
||||
func New(username, password, mechanism, service, host string) (Stepper, error) {
|
||||
initOnce.Do(initSASL)
|
||||
if initError != nil {
|
||||
return nil, initError
|
||||
}
|
||||
|
||||
ss := &saslSession{mech: mechanism}
|
||||
if service == "" {
|
||||
service = "mongodb"
|
||||
}
|
||||
if i := strings.Index(host, ":"); i >= 0 {
|
||||
host = host[:i]
|
||||
}
|
||||
ss.callbacks = C.mgo_sasl_callbacks(ss.cstr(username), ss.cstr(password))
|
||||
rc := C.sasl_client_new(ss.cstr(service), ss.cstr(host), nil, nil, ss.callbacks, 0, &ss.conn)
|
||||
if rc != C.SASL_OK {
|
||||
ss.Close()
|
||||
return nil, saslError(rc, nil, "cannot create new SASL client")
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
func (ss *saslSession) cstr(s string) *C.char {
|
||||
cstr := C.CString(s)
|
||||
ss.cstrings = append(ss.cstrings, cstr)
|
||||
return cstr
|
||||
}
|
||||
|
||||
func (ss *saslSession) Close() {
|
||||
for _, cstr := range ss.cstrings {
|
||||
C.free(unsafe.Pointer(cstr))
|
||||
}
|
||||
ss.cstrings = nil
|
||||
|
||||
if ss.callbacks != nil {
|
||||
C.free(unsafe.Pointer(ss.callbacks))
|
||||
}
|
||||
|
||||
// The documentation of SASL dispose makes it clear that this should only
|
||||
// be done when the connection is done, not when the authentication phase
|
||||
// is done, because an encryption layer may have been negotiated.
|
||||
// Even then, we'll do this for now, because it's simpler and prevents
|
||||
// keeping track of this state for every socket. If it breaks, we'll fix it.
|
||||
C.sasl_dispose(&ss.conn)
|
||||
}
|
||||
|
||||
func (ss *saslSession) Step(serverData []byte) (clientData []byte, done bool, err error) {
|
||||
ss.step++
|
||||
if ss.step > 10 {
|
||||
return nil, false, fmt.Errorf("too many SASL steps without authentication")
|
||||
}
|
||||
var cclientData *C.char
|
||||
var cclientDataLen C.uint
|
||||
var rc C.int
|
||||
if ss.step == 1 {
|
||||
var mechanism *C.char // ignored - must match cred
|
||||
rc = C.sasl_client_start(ss.conn, ss.cstr(ss.mech), nil, &cclientData, &cclientDataLen, &mechanism)
|
||||
} else {
|
||||
var cserverData *C.char
|
||||
var cserverDataLen C.uint
|
||||
if len(serverData) > 0 {
|
||||
cserverData = (*C.char)(unsafe.Pointer(&serverData[0]))
|
||||
cserverDataLen = C.uint(len(serverData))
|
||||
}
|
||||
rc = C.sasl_client_step(ss.conn, cserverData, cserverDataLen, nil, &cclientData, &cclientDataLen)
|
||||
}
|
||||
if cclientData != nil && cclientDataLen > 0 {
|
||||
clientData = C.GoBytes(unsafe.Pointer(cclientData), C.int(cclientDataLen))
|
||||
}
|
||||
if rc == C.SASL_OK {
|
||||
return clientData, true, nil
|
||||
}
|
||||
if rc == C.SASL_CONTINUE {
|
||||
return clientData, false, nil
|
||||
}
|
||||
|
||||
return nil, false, saslError(rc, ss.conn, "cannot establish SASL session")
|
||||
}
|
||||
|
||||
func saslError(rc C.int, conn *C.sasl_conn_t, msg string) error {
|
||||
var detail string
|
||||
if conn == nil {
|
||||
detail = C.GoString(C.sasl_errstring(rc, nil, nil))
|
||||
} else {
|
||||
detail = C.GoString(C.sasl_errdetail(conn))
|
||||
}
|
||||
return fmt.Errorf(msg + ": " + detail)
|
||||
}
|
||||
-122
@@ -1,122 +0,0 @@
|
||||
#include "sasl_windows.h"
|
||||
|
||||
static const LPSTR SSPI_PACKAGE_NAME = "kerberos";
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY sspi_acquire_credentials_handle(CredHandle *cred_handle, char *username, char *password, char *domain)
|
||||
{
|
||||
SEC_WINNT_AUTH_IDENTITY auth_identity;
|
||||
SECURITY_INTEGER ignored;
|
||||
|
||||
auth_identity.Flags = SEC_WINNT_AUTH_IDENTITY_ANSI;
|
||||
auth_identity.User = (LPSTR) username;
|
||||
auth_identity.UserLength = strlen(username);
|
||||
auth_identity.Password = NULL;
|
||||
auth_identity.PasswordLength = 0;
|
||||
if(password){
|
||||
auth_identity.Password = (LPSTR) password;
|
||||
auth_identity.PasswordLength = strlen(password);
|
||||
}
|
||||
auth_identity.Domain = (LPSTR) domain;
|
||||
auth_identity.DomainLength = strlen(domain);
|
||||
return call_sspi_acquire_credentials_handle(NULL, SSPI_PACKAGE_NAME, SECPKG_CRED_OUTBOUND, NULL, &auth_identity, NULL, NULL, cred_handle, &ignored);
|
||||
}
|
||||
|
||||
int sspi_step(CredHandle *cred_handle, int has_context, CtxtHandle *context, PVOID buffer, ULONG buffer_length, PVOID *out_buffer, ULONG *out_buffer_length, char *target)
|
||||
{
|
||||
SecBufferDesc inbuf;
|
||||
SecBuffer in_bufs[1];
|
||||
SecBufferDesc outbuf;
|
||||
SecBuffer out_bufs[1];
|
||||
|
||||
if (has_context > 0) {
|
||||
// If we already have a context, we now have data to send.
|
||||
// Put this data in an inbuf.
|
||||
inbuf.ulVersion = SECBUFFER_VERSION;
|
||||
inbuf.cBuffers = 1;
|
||||
inbuf.pBuffers = in_bufs;
|
||||
in_bufs[0].pvBuffer = buffer;
|
||||
in_bufs[0].cbBuffer = buffer_length;
|
||||
in_bufs[0].BufferType = SECBUFFER_TOKEN;
|
||||
}
|
||||
|
||||
outbuf.ulVersion = SECBUFFER_VERSION;
|
||||
outbuf.cBuffers = 1;
|
||||
outbuf.pBuffers = out_bufs;
|
||||
out_bufs[0].pvBuffer = NULL;
|
||||
out_bufs[0].cbBuffer = 0;
|
||||
out_bufs[0].BufferType = SECBUFFER_TOKEN;
|
||||
|
||||
ULONG context_attr = 0;
|
||||
|
||||
int ret = call_sspi_initialize_security_context(cred_handle,
|
||||
has_context > 0 ? context : NULL,
|
||||
(LPSTR) target,
|
||||
ISC_REQ_ALLOCATE_MEMORY | ISC_REQ_MUTUAL_AUTH,
|
||||
0,
|
||||
SECURITY_NETWORK_DREP,
|
||||
has_context > 0 ? &inbuf : NULL,
|
||||
0,
|
||||
context,
|
||||
&outbuf,
|
||||
&context_attr,
|
||||
NULL);
|
||||
|
||||
*out_buffer = malloc(out_bufs[0].cbBuffer);
|
||||
*out_buffer_length = out_bufs[0].cbBuffer;
|
||||
memcpy(*out_buffer, out_bufs[0].pvBuffer, *out_buffer_length);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int sspi_send_client_authz_id(CtxtHandle *context, PVOID *buffer, ULONG *buffer_length, char *user_plus_realm)
|
||||
{
|
||||
SecPkgContext_Sizes sizes;
|
||||
SECURITY_STATUS status = call_sspi_query_context_attributes(context, SECPKG_ATTR_SIZES, &sizes);
|
||||
|
||||
if (status != SEC_E_OK) {
|
||||
return status;
|
||||
}
|
||||
|
||||
size_t user_plus_realm_length = strlen(user_plus_realm);
|
||||
int msgSize = 4 + user_plus_realm_length;
|
||||
char *msg = malloc((sizes.cbSecurityTrailer + msgSize + sizes.cbBlockSize) * sizeof(char));
|
||||
msg[sizes.cbSecurityTrailer + 0] = 1;
|
||||
msg[sizes.cbSecurityTrailer + 1] = 0;
|
||||
msg[sizes.cbSecurityTrailer + 2] = 0;
|
||||
msg[sizes.cbSecurityTrailer + 3] = 0;
|
||||
memcpy(&msg[sizes.cbSecurityTrailer + 4], user_plus_realm, user_plus_realm_length);
|
||||
|
||||
SecBuffer wrapBufs[3];
|
||||
SecBufferDesc wrapBufDesc;
|
||||
wrapBufDesc.cBuffers = 3;
|
||||
wrapBufDesc.pBuffers = wrapBufs;
|
||||
wrapBufDesc.ulVersion = SECBUFFER_VERSION;
|
||||
|
||||
wrapBufs[0].cbBuffer = sizes.cbSecurityTrailer;
|
||||
wrapBufs[0].BufferType = SECBUFFER_TOKEN;
|
||||
wrapBufs[0].pvBuffer = msg;
|
||||
|
||||
wrapBufs[1].cbBuffer = msgSize;
|
||||
wrapBufs[1].BufferType = SECBUFFER_DATA;
|
||||
wrapBufs[1].pvBuffer = msg + sizes.cbSecurityTrailer;
|
||||
|
||||
wrapBufs[2].cbBuffer = sizes.cbBlockSize;
|
||||
wrapBufs[2].BufferType = SECBUFFER_PADDING;
|
||||
wrapBufs[2].pvBuffer = msg + sizes.cbSecurityTrailer + msgSize;
|
||||
|
||||
status = call_sspi_encrypt_message(context, SECQOP_WRAP_NO_ENCRYPT, &wrapBufDesc, 0);
|
||||
if (status != SEC_E_OK) {
|
||||
free(msg);
|
||||
return status;
|
||||
}
|
||||
|
||||
*buffer_length = wrapBufs[0].cbBuffer + wrapBufs[1].cbBuffer + wrapBufs[2].cbBuffer;
|
||||
*buffer = malloc(*buffer_length);
|
||||
|
||||
memcpy(*buffer, wrapBufs[0].pvBuffer, wrapBufs[0].cbBuffer);
|
||||
memcpy(*buffer + wrapBufs[0].cbBuffer, wrapBufs[1].pvBuffer, wrapBufs[1].cbBuffer);
|
||||
memcpy(*buffer + wrapBufs[0].cbBuffer + wrapBufs[1].cbBuffer, wrapBufs[2].pvBuffer, wrapBufs[2].cbBuffer);
|
||||
|
||||
free(msg);
|
||||
return SEC_E_OK;
|
||||
}
|
||||
-142
@@ -1,142 +0,0 @@
|
||||
package sasl
|
||||
|
||||
// #include "sasl_windows.h"
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type saslStepper interface {
|
||||
Step(serverData []byte) (clientData []byte, done bool, err error)
|
||||
Close()
|
||||
}
|
||||
|
||||
type saslSession struct {
|
||||
// Credentials
|
||||
mech string
|
||||
service string
|
||||
host string
|
||||
userPlusRealm string
|
||||
target string
|
||||
domain string
|
||||
|
||||
// Internal state
|
||||
authComplete bool
|
||||
errored bool
|
||||
step int
|
||||
|
||||
// C internal state
|
||||
credHandle C.CredHandle
|
||||
context C.CtxtHandle
|
||||
hasContext C.int
|
||||
|
||||
// Keep track of pointers we need to explicitly free
|
||||
stringsToFree []*C.char
|
||||
}
|
||||
|
||||
var initError error
|
||||
var initOnce sync.Once
|
||||
|
||||
func initSSPI() {
|
||||
rc := C.load_secur32_dll()
|
||||
if rc != 0 {
|
||||
initError = fmt.Errorf("Error loading libraries: %v", rc)
|
||||
}
|
||||
}
|
||||
|
||||
func New(username, password, mechanism, service, host string) (saslStepper, error) {
|
||||
initOnce.Do(initSSPI)
|
||||
ss := &saslSession{mech: mechanism, hasContext: 0, userPlusRealm: username}
|
||||
if service == "" {
|
||||
service = "mongodb"
|
||||
}
|
||||
if i := strings.Index(host, ":"); i >= 0 {
|
||||
host = host[:i]
|
||||
}
|
||||
ss.service = service
|
||||
ss.host = host
|
||||
|
||||
usernameComponents := strings.Split(username, "@")
|
||||
if len(usernameComponents) < 2 {
|
||||
return nil, fmt.Errorf("Username '%v' doesn't contain a realm!", username)
|
||||
}
|
||||
user := usernameComponents[0]
|
||||
ss.domain = usernameComponents[1]
|
||||
ss.target = fmt.Sprintf("%s/%s", ss.service, ss.host)
|
||||
|
||||
var status C.SECURITY_STATUS
|
||||
// Step 0: call AcquireCredentialsHandle to get a nice SSPI CredHandle
|
||||
if len(password) > 0 {
|
||||
status = C.sspi_acquire_credentials_handle(&ss.credHandle, ss.cstr(user), ss.cstr(password), ss.cstr(ss.domain))
|
||||
} else {
|
||||
status = C.sspi_acquire_credentials_handle(&ss.credHandle, ss.cstr(user), nil, ss.cstr(ss.domain))
|
||||
}
|
||||
if status != C.SEC_E_OK {
|
||||
ss.errored = true
|
||||
return nil, fmt.Errorf("Couldn't create new SSPI client, error code %v", status)
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
func (ss *saslSession) cstr(s string) *C.char {
|
||||
cstr := C.CString(s)
|
||||
ss.stringsToFree = append(ss.stringsToFree, cstr)
|
||||
return cstr
|
||||
}
|
||||
|
||||
func (ss *saslSession) Close() {
|
||||
for _, cstr := range ss.stringsToFree {
|
||||
C.free(unsafe.Pointer(cstr))
|
||||
}
|
||||
}
|
||||
|
||||
func (ss *saslSession) Step(serverData []byte) (clientData []byte, done bool, err error) {
|
||||
ss.step++
|
||||
if ss.step > 10 {
|
||||
return nil, false, fmt.Errorf("too many SSPI steps without authentication")
|
||||
}
|
||||
var buffer C.PVOID
|
||||
var bufferLength C.ULONG
|
||||
var outBuffer C.PVOID
|
||||
var outBufferLength C.ULONG
|
||||
if len(serverData) > 0 {
|
||||
buffer = (C.PVOID)(unsafe.Pointer(&serverData[0]))
|
||||
bufferLength = C.ULONG(len(serverData))
|
||||
}
|
||||
var status C.int
|
||||
if ss.authComplete {
|
||||
// Step 3: last bit of magic to use the correct server credentials
|
||||
status = C.sspi_send_client_authz_id(&ss.context, &outBuffer, &outBufferLength, ss.cstr(ss.userPlusRealm))
|
||||
} else {
|
||||
// Step 1 + Step 2: set up security context with the server and TGT
|
||||
status = C.sspi_step(&ss.credHandle, ss.hasContext, &ss.context, buffer, bufferLength, &outBuffer, &outBufferLength, ss.cstr(ss.target))
|
||||
}
|
||||
if outBuffer != C.PVOID(nil) {
|
||||
defer C.free(unsafe.Pointer(outBuffer))
|
||||
}
|
||||
if status != C.SEC_E_OK && status != C.SEC_I_CONTINUE_NEEDED {
|
||||
ss.errored = true
|
||||
return nil, false, ss.handleSSPIErrorCode(status)
|
||||
}
|
||||
|
||||
clientData = C.GoBytes(unsafe.Pointer(outBuffer), C.int(outBufferLength))
|
||||
if status == C.SEC_E_OK {
|
||||
ss.authComplete = true
|
||||
return clientData, true, nil
|
||||
} else {
|
||||
ss.hasContext = 1
|
||||
return clientData, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (ss *saslSession) handleSSPIErrorCode(code C.int) error {
|
||||
switch {
|
||||
case code == C.SEC_E_TARGET_UNKNOWN:
|
||||
return fmt.Errorf("Target %v@%v not found", ss.target, ss.domain)
|
||||
}
|
||||
return fmt.Errorf("Unknown error doing step %v, error code %v", ss.step, code)
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
#include <windows.h>
|
||||
|
||||
#include "sspi_windows.h"
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY sspi_acquire_credentials_handle(CredHandle* cred_handle, char* username, char* password, char* domain);
|
||||
int sspi_step(CredHandle* cred_handle, int has_context, CtxtHandle* context, PVOID buffer, ULONG buffer_length, PVOID* out_buffer, ULONG* out_buffer_length, char* target);
|
||||
int sspi_send_client_authz_id(CtxtHandle* context, PVOID* buffer, ULONG* buffer_length, char* user_plus_realm);
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
// Code adapted from the NodeJS kerberos library:
|
||||
//
|
||||
// https://github.com/christkv/kerberos/tree/master/lib/win32/kerberos_sspi.c
|
||||
//
|
||||
// Under the terms of the Apache License, Version 2.0:
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "sspi_windows.h"
|
||||
|
||||
static HINSTANCE sspi_secur32_dll = NULL;
|
||||
|
||||
int load_secur32_dll()
|
||||
{
|
||||
sspi_secur32_dll = LoadLibrary("secur32.dll");
|
||||
if (sspi_secur32_dll == NULL) {
|
||||
return GetLastError();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_encrypt_message(PCtxtHandle phContext, unsigned long fQOP, PSecBufferDesc pMessage, unsigned long MessageSeqNo)
|
||||
{
|
||||
if (sspi_secur32_dll == NULL) {
|
||||
return -1;
|
||||
}
|
||||
encryptMessage_fn pfn_encryptMessage = (encryptMessage_fn) GetProcAddress(sspi_secur32_dll, "EncryptMessage");
|
||||
if (!pfn_encryptMessage) {
|
||||
return -2;
|
||||
}
|
||||
return (*pfn_encryptMessage)(phContext, fQOP, pMessage, MessageSeqNo);
|
||||
}
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_acquire_credentials_handle(
|
||||
LPSTR pszPrincipal, LPSTR pszPackage, unsigned long fCredentialUse,
|
||||
void *pvLogonId, void *pAuthData, SEC_GET_KEY_FN pGetKeyFn, void *pvGetKeyArgument,
|
||||
PCredHandle phCredential, PTimeStamp ptsExpiry)
|
||||
{
|
||||
if (sspi_secur32_dll == NULL) {
|
||||
return -1;
|
||||
}
|
||||
acquireCredentialsHandle_fn pfn_acquireCredentialsHandle;
|
||||
#ifdef _UNICODE
|
||||
pfn_acquireCredentialsHandle = (acquireCredentialsHandle_fn) GetProcAddress(sspi_secur32_dll, "AcquireCredentialsHandleW");
|
||||
#else
|
||||
pfn_acquireCredentialsHandle = (acquireCredentialsHandle_fn) GetProcAddress(sspi_secur32_dll, "AcquireCredentialsHandleA");
|
||||
#endif
|
||||
if (!pfn_acquireCredentialsHandle) {
|
||||
return -2;
|
||||
}
|
||||
return (*pfn_acquireCredentialsHandle)(
|
||||
pszPrincipal, pszPackage, fCredentialUse, pvLogonId, pAuthData,
|
||||
pGetKeyFn, pvGetKeyArgument, phCredential, ptsExpiry);
|
||||
}
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_initialize_security_context(
|
||||
PCredHandle phCredential, PCtxtHandle phContext, LPSTR pszTargetName,
|
||||
unsigned long fContextReq, unsigned long Reserved1, unsigned long TargetDataRep,
|
||||
PSecBufferDesc pInput, unsigned long Reserved2, PCtxtHandle phNewContext,
|
||||
PSecBufferDesc pOutput, unsigned long *pfContextAttr, PTimeStamp ptsExpiry)
|
||||
{
|
||||
if (sspi_secur32_dll == NULL) {
|
||||
return -1;
|
||||
}
|
||||
initializeSecurityContext_fn pfn_initializeSecurityContext;
|
||||
#ifdef _UNICODE
|
||||
pfn_initializeSecurityContext = (initializeSecurityContext_fn) GetProcAddress(sspi_secur32_dll, "InitializeSecurityContextW");
|
||||
#else
|
||||
pfn_initializeSecurityContext = (initializeSecurityContext_fn) GetProcAddress(sspi_secur32_dll, "InitializeSecurityContextA");
|
||||
#endif
|
||||
if (!pfn_initializeSecurityContext) {
|
||||
return -2;
|
||||
}
|
||||
return (*pfn_initializeSecurityContext)(
|
||||
phCredential, phContext, pszTargetName, fContextReq, Reserved1, TargetDataRep,
|
||||
pInput, Reserved2, phNewContext, pOutput, pfContextAttr, ptsExpiry);
|
||||
}
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_query_context_attributes(PCtxtHandle phContext, unsigned long ulAttribute, void *pBuffer)
|
||||
{
|
||||
if (sspi_secur32_dll == NULL) {
|
||||
return -1;
|
||||
}
|
||||
queryContextAttributes_fn pfn_queryContextAttributes;
|
||||
#ifdef _UNICODE
|
||||
pfn_queryContextAttributes = (queryContextAttributes_fn) GetProcAddress(sspi_secur32_dll, "QueryContextAttributesW");
|
||||
#else
|
||||
pfn_queryContextAttributes = (queryContextAttributes_fn) GetProcAddress(sspi_secur32_dll, "QueryContextAttributesA");
|
||||
#endif
|
||||
if (!pfn_queryContextAttributes) {
|
||||
return -2;
|
||||
}
|
||||
return (*pfn_queryContextAttributes)(phContext, ulAttribute, pBuffer);
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
// Code adapted from the NodeJS kerberos library:
|
||||
//
|
||||
// https://github.com/christkv/kerberos/tree/master/lib/win32/kerberos_sspi.h
|
||||
//
|
||||
// Under the terms of the Apache License, Version 2.0:
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
#ifndef SSPI_WINDOWS_H
|
||||
#define SSPI_WINDOWS_H
|
||||
|
||||
#define SECURITY_WIN32 1
|
||||
|
||||
#include <windows.h>
|
||||
#include <sspi.h>
|
||||
|
||||
int load_secur32_dll();
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_encrypt_message(PCtxtHandle phContext, unsigned long fQOP, PSecBufferDesc pMessage, unsigned long MessageSeqNo);
|
||||
|
||||
typedef DWORD (WINAPI *encryptMessage_fn)(PCtxtHandle phContext, ULONG fQOP, PSecBufferDesc pMessage, ULONG MessageSeqNo);
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_acquire_credentials_handle(
|
||||
LPSTR pszPrincipal, // Name of principal
|
||||
LPSTR pszPackage, // Name of package
|
||||
unsigned long fCredentialUse, // Flags indicating use
|
||||
void *pvLogonId, // Pointer to logon ID
|
||||
void *pAuthData, // Package specific data
|
||||
SEC_GET_KEY_FN pGetKeyFn, // Pointer to GetKey() func
|
||||
void *pvGetKeyArgument, // Value to pass to GetKey()
|
||||
PCredHandle phCredential, // (out) Cred Handle
|
||||
PTimeStamp ptsExpiry // (out) Lifetime (optional)
|
||||
);
|
||||
|
||||
typedef DWORD (WINAPI *acquireCredentialsHandle_fn)(
|
||||
LPSTR pszPrincipal, LPSTR pszPackage, unsigned long fCredentialUse,
|
||||
void *pvLogonId, void *pAuthData, SEC_GET_KEY_FN pGetKeyFn, void *pvGetKeyArgument,
|
||||
PCredHandle phCredential, PTimeStamp ptsExpiry
|
||||
);
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_initialize_security_context(
|
||||
PCredHandle phCredential, // Cred to base context
|
||||
PCtxtHandle phContext, // Existing context (OPT)
|
||||
LPSTR pszTargetName, // Name of target
|
||||
unsigned long fContextReq, // Context Requirements
|
||||
unsigned long Reserved1, // Reserved, MBZ
|
||||
unsigned long TargetDataRep, // Data rep of target
|
||||
PSecBufferDesc pInput, // Input Buffers
|
||||
unsigned long Reserved2, // Reserved, MBZ
|
||||
PCtxtHandle phNewContext, // (out) New Context handle
|
||||
PSecBufferDesc pOutput, // (inout) Output Buffers
|
||||
unsigned long *pfContextAttr, // (out) Context attrs
|
||||
PTimeStamp ptsExpiry // (out) Life span (OPT)
|
||||
);
|
||||
|
||||
typedef DWORD (WINAPI *initializeSecurityContext_fn)(
|
||||
PCredHandle phCredential, PCtxtHandle phContext, LPSTR pszTargetName, unsigned long fContextReq,
|
||||
unsigned long Reserved1, unsigned long TargetDataRep, PSecBufferDesc pInput, unsigned long Reserved2,
|
||||
PCtxtHandle phNewContext, PSecBufferDesc pOutput, unsigned long *pfContextAttr, PTimeStamp ptsExpiry);
|
||||
|
||||
SECURITY_STATUS SEC_ENTRY call_sspi_query_context_attributes(
|
||||
PCtxtHandle phContext, // Context to query
|
||||
unsigned long ulAttribute, // Attribute to query
|
||||
void *pBuffer // Buffer for attributes
|
||||
);
|
||||
|
||||
typedef DWORD (WINAPI *queryContextAttributes_fn)(
|
||||
PCtxtHandle phContext, unsigned long ulAttribute, void *pBuffer);
|
||||
|
||||
#endif // SSPI_WINDOWS_H
|
||||
-266
@@ -1,266 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2014 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
// Package scram implements a SCRAM-{SHA-1,etc} client per RFC5802.
|
||||
//
|
||||
// http://tools.ietf.org/html/rfc5802
|
||||
//
|
||||
package scram
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"hash"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Client implements a SCRAM-* client (SCRAM-SHA-1, SCRAM-SHA-256, etc).
|
||||
//
|
||||
// A Client may be used within a SASL conversation with logic resembling:
|
||||
//
|
||||
// var in []byte
|
||||
// var client = scram.NewClient(sha1.New, user, pass)
|
||||
// for client.Step(in) {
|
||||
// out := client.Out()
|
||||
// // send out to server
|
||||
// in := serverOut
|
||||
// }
|
||||
// if client.Err() != nil {
|
||||
// // auth failed
|
||||
// }
|
||||
//
|
||||
type Client struct {
|
||||
newHash func() hash.Hash
|
||||
|
||||
user string
|
||||
pass string
|
||||
step int
|
||||
out bytes.Buffer
|
||||
err error
|
||||
|
||||
clientNonce []byte
|
||||
serverNonce []byte
|
||||
saltedPass []byte
|
||||
authMsg bytes.Buffer
|
||||
}
|
||||
|
||||
// NewClient returns a new SCRAM-* client with the provided hash algorithm.
|
||||
//
|
||||
// For SCRAM-SHA-1, for example, use:
|
||||
//
|
||||
// client := scram.NewClient(sha1.New, user, pass)
|
||||
//
|
||||
func NewClient(newHash func() hash.Hash, user, pass string) *Client {
|
||||
c := &Client{
|
||||
newHash: newHash,
|
||||
user: user,
|
||||
pass: pass,
|
||||
}
|
||||
c.out.Grow(256)
|
||||
c.authMsg.Grow(256)
|
||||
return c
|
||||
}
|
||||
|
||||
// Out returns the data to be sent to the server in the current step.
|
||||
func (c *Client) Out() []byte {
|
||||
if c.out.Len() == 0 {
|
||||
return []byte{}
|
||||
}
|
||||
return c.out.Bytes()
|
||||
}
|
||||
|
||||
// Err returns the error that occurred, or nil if there were no errors.
|
||||
func (c *Client) Err() error {
|
||||
return c.err
|
||||
}
|
||||
|
||||
// SetNonce sets the client nonce to the provided value.
|
||||
// If not set, the nonce is generated automatically out of crypto/rand on the first step.
|
||||
func (c *Client) SetNonce(nonce []byte) {
|
||||
c.clientNonce = nonce
|
||||
}
|
||||
|
||||
var escaper = strings.NewReplacer("=", "=3D", ",", "=2C")
|
||||
|
||||
// Step processes the incoming data from the server and makes the
|
||||
// next round of data for the server available via Client.Out.
|
||||
// Step returns false if there are no errors and more data is
|
||||
// still expected.
|
||||
func (c *Client) Step(in []byte) bool {
|
||||
c.out.Reset()
|
||||
if c.step > 2 || c.err != nil {
|
||||
return false
|
||||
}
|
||||
c.step++
|
||||
switch c.step {
|
||||
case 1:
|
||||
c.err = c.step1(in)
|
||||
case 2:
|
||||
c.err = c.step2(in)
|
||||
case 3:
|
||||
c.err = c.step3(in)
|
||||
}
|
||||
return c.step > 2 || c.err != nil
|
||||
}
|
||||
|
||||
func (c *Client) step1(in []byte) error {
|
||||
if len(c.clientNonce) == 0 {
|
||||
const nonceLen = 6
|
||||
buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen))
|
||||
if _, err := rand.Read(buf[:nonceLen]); err != nil {
|
||||
return fmt.Errorf("cannot read random SCRAM-SHA-1 nonce from operating system: %v", err)
|
||||
}
|
||||
c.clientNonce = buf[nonceLen:]
|
||||
b64.Encode(c.clientNonce, buf[:nonceLen])
|
||||
}
|
||||
c.authMsg.WriteString("n=")
|
||||
escaper.WriteString(&c.authMsg, c.user)
|
||||
c.authMsg.WriteString(",r=")
|
||||
c.authMsg.Write(c.clientNonce)
|
||||
|
||||
c.out.WriteString("n,,")
|
||||
c.out.Write(c.authMsg.Bytes())
|
||||
return nil
|
||||
}
|
||||
|
||||
var b64 = base64.StdEncoding
|
||||
|
||||
func (c *Client) step2(in []byte) error {
|
||||
c.authMsg.WriteByte(',')
|
||||
c.authMsg.Write(in)
|
||||
|
||||
fields := bytes.Split(in, []byte(","))
|
||||
if len(fields) != 3 {
|
||||
return fmt.Errorf("expected 3 fields in first SCRAM-SHA-1 server message, got %d: %q", len(fields), in)
|
||||
}
|
||||
if !bytes.HasPrefix(fields[0], []byte("r=")) || len(fields[0]) < 2 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 nonce: %q", fields[0])
|
||||
}
|
||||
if !bytes.HasPrefix(fields[1], []byte("s=")) || len(fields[1]) < 6 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 salt: %q", fields[1])
|
||||
}
|
||||
if !bytes.HasPrefix(fields[2], []byte("i=")) || len(fields[2]) < 6 {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 iteration count: %q", fields[2])
|
||||
}
|
||||
|
||||
c.serverNonce = fields[0][2:]
|
||||
if !bytes.HasPrefix(c.serverNonce, c.clientNonce) {
|
||||
return fmt.Errorf("server SCRAM-SHA-1 nonce is not prefixed by client nonce: got %q, want %q+\"...\"", c.serverNonce, c.clientNonce)
|
||||
}
|
||||
|
||||
salt := make([]byte, b64.DecodedLen(len(fields[1][2:])))
|
||||
n, err := b64.Decode(salt, fields[1][2:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decode SCRAM-SHA-1 salt sent by server: %q", fields[1])
|
||||
}
|
||||
salt = salt[:n]
|
||||
iterCount, err := strconv.Atoi(string(fields[2][2:]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("server sent an invalid SCRAM-SHA-1 iteration count: %q", fields[2])
|
||||
}
|
||||
c.saltPassword(salt, iterCount)
|
||||
|
||||
c.authMsg.WriteString(",c=biws,r=")
|
||||
c.authMsg.Write(c.serverNonce)
|
||||
|
||||
c.out.WriteString("c=biws,r=")
|
||||
c.out.Write(c.serverNonce)
|
||||
c.out.WriteString(",p=")
|
||||
c.out.Write(c.clientProof())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) step3(in []byte) error {
|
||||
var isv, ise bool
|
||||
var fields = bytes.Split(in, []byte(","))
|
||||
if len(fields) == 1 {
|
||||
isv = bytes.HasPrefix(fields[0], []byte("v="))
|
||||
ise = bytes.HasPrefix(fields[0], []byte("e="))
|
||||
}
|
||||
if ise {
|
||||
return fmt.Errorf("SCRAM-SHA-1 authentication error: %s", fields[0][2:])
|
||||
} else if !isv {
|
||||
return fmt.Errorf("unsupported SCRAM-SHA-1 final message from server: %q", in)
|
||||
}
|
||||
if !bytes.Equal(c.serverSignature(), fields[0][2:]) {
|
||||
return fmt.Errorf("cannot authenticate SCRAM-SHA-1 server signature: %q", fields[0][2:])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) saltPassword(salt []byte, iterCount int) {
|
||||
mac := hmac.New(c.newHash, []byte(c.pass))
|
||||
mac.Write(salt)
|
||||
mac.Write([]byte{0, 0, 0, 1})
|
||||
ui := mac.Sum(nil)
|
||||
hi := make([]byte, len(ui))
|
||||
copy(hi, ui)
|
||||
for i := 1; i < iterCount; i++ {
|
||||
mac.Reset()
|
||||
mac.Write(ui)
|
||||
mac.Sum(ui[:0])
|
||||
for j, b := range ui {
|
||||
hi[j] ^= b
|
||||
}
|
||||
}
|
||||
c.saltedPass = hi
|
||||
}
|
||||
|
||||
func (c *Client) clientProof() []byte {
|
||||
mac := hmac.New(c.newHash, c.saltedPass)
|
||||
mac.Write([]byte("Client Key"))
|
||||
clientKey := mac.Sum(nil)
|
||||
hash := c.newHash()
|
||||
hash.Write(clientKey)
|
||||
storedKey := hash.Sum(nil)
|
||||
mac = hmac.New(c.newHash, storedKey)
|
||||
mac.Write(c.authMsg.Bytes())
|
||||
clientProof := mac.Sum(nil)
|
||||
for i, b := range clientKey {
|
||||
clientProof[i] ^= b
|
||||
}
|
||||
clientProof64 := make([]byte, b64.EncodedLen(len(clientProof)))
|
||||
b64.Encode(clientProof64, clientProof)
|
||||
return clientProof64
|
||||
}
|
||||
|
||||
func (c *Client) serverSignature() []byte {
|
||||
mac := hmac.New(c.newHash, c.saltedPass)
|
||||
mac.Write([]byte("Server Key"))
|
||||
serverKey := mac.Sum(nil)
|
||||
|
||||
mac = hmac.New(c.newHash, serverKey)
|
||||
mac.Write(c.authMsg.Bytes())
|
||||
serverSignature := mac.Sum(nil)
|
||||
|
||||
encoded := make([]byte, b64.EncodedLen(len(serverSignature)))
|
||||
b64.Encode(encoded, serverSignature)
|
||||
return encoded
|
||||
}
|
||||
-133
@@ -1,133 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Logging integration.
|
||||
|
||||
// LogLogger avoid importing the log type information unnecessarily. There's a small cost
|
||||
// associated with using an interface rather than the type. Depending on how
|
||||
// often the logger is plugged in, it would be worth using the type instead.
|
||||
type logLogger interface {
|
||||
Output(calldepth int, s string) error
|
||||
}
|
||||
|
||||
var (
|
||||
globalLogger logLogger
|
||||
globalDebug bool
|
||||
globalMutex sync.Mutex
|
||||
)
|
||||
|
||||
// RACE WARNING: There are known data races when logging, which are manually
|
||||
// silenced when the race detector is in use. These data races won't be
|
||||
// observed in typical use, because logging is supposed to be set up once when
|
||||
// the application starts. Having raceDetector as a constant, the compiler
|
||||
// should elide the locks altogether in actual use.
|
||||
|
||||
// SetLogger specify the *log.Logger object where log messages should be sent to.
|
||||
func SetLogger(logger logLogger) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
globalLogger = logger
|
||||
}
|
||||
|
||||
// SetDebug enable the delivery of debug messages to the logger. Only meaningful
|
||||
// if a logger is also set.
|
||||
func SetDebug(debug bool) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
globalDebug = debug
|
||||
}
|
||||
|
||||
func log(v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprint(v...))
|
||||
}
|
||||
}
|
||||
|
||||
func logln(v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprintln(v...))
|
||||
}
|
||||
}
|
||||
|
||||
func logf(format string, v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprintf(format, v...))
|
||||
}
|
||||
}
|
||||
|
||||
func debug(v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalDebug && globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprint(v...))
|
||||
}
|
||||
}
|
||||
|
||||
func debugln(v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalDebug && globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprintln(v...))
|
||||
}
|
||||
}
|
||||
|
||||
func debugf(format string, v ...interface{}) {
|
||||
if raceDetector {
|
||||
globalMutex.Lock()
|
||||
defer globalMutex.Unlock()
|
||||
}
|
||||
if globalDebug && globalLogger != nil {
|
||||
globalLogger.Output(2, fmt.Sprintf(format, v...))
|
||||
}
|
||||
}
|
||||
-91
@@ -1,91 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
type queue struct {
|
||||
elems []interface{}
|
||||
nelems, popi, pushi int
|
||||
}
|
||||
|
||||
func (q *queue) Len() int {
|
||||
return q.nelems
|
||||
}
|
||||
|
||||
func (q *queue) Push(elem interface{}) {
|
||||
//debugf("Pushing(pushi=%d popi=%d cap=%d): %#v\n",
|
||||
// q.pushi, q.popi, len(q.elems), elem)
|
||||
if q.nelems == len(q.elems) {
|
||||
q.expand()
|
||||
}
|
||||
q.elems[q.pushi] = elem
|
||||
q.nelems++
|
||||
q.pushi = (q.pushi + 1) % len(q.elems)
|
||||
//debugf(" Pushed(pushi=%d popi=%d cap=%d): %#v\n",
|
||||
// q.pushi, q.popi, len(q.elems), elem)
|
||||
}
|
||||
|
||||
func (q *queue) Pop() (elem interface{}) {
|
||||
//debugf("Popping(pushi=%d popi=%d cap=%d)\n",
|
||||
// q.pushi, q.popi, len(q.elems))
|
||||
if q.nelems == 0 {
|
||||
return nil
|
||||
}
|
||||
elem = q.elems[q.popi]
|
||||
q.elems[q.popi] = nil // Help GC.
|
||||
q.nelems--
|
||||
q.popi = (q.popi + 1) % len(q.elems)
|
||||
//debugf(" Popped(pushi=%d popi=%d cap=%d): %#v\n",
|
||||
// q.pushi, q.popi, len(q.elems), elem)
|
||||
return elem
|
||||
}
|
||||
|
||||
func (q *queue) expand() {
|
||||
curcap := len(q.elems)
|
||||
var newcap int
|
||||
if curcap == 0 {
|
||||
newcap = 8
|
||||
} else if curcap < 1024 {
|
||||
newcap = curcap * 2
|
||||
} else {
|
||||
newcap = curcap + (curcap / 4)
|
||||
}
|
||||
elems := make([]interface{}, newcap)
|
||||
|
||||
if q.popi == 0 {
|
||||
copy(elems, q.elems)
|
||||
q.pushi = curcap
|
||||
} else {
|
||||
newpopi := newcap - (curcap - q.popi)
|
||||
copy(elems, q.elems[:q.popi])
|
||||
copy(elems[newpopi:], q.elems[q.popi:])
|
||||
q.popi = newpopi
|
||||
}
|
||||
for i := range q.elems {
|
||||
q.elems[i] = nil // Help GC.
|
||||
}
|
||||
q.elems = elems
|
||||
}
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
// +build !race
|
||||
|
||||
package mgo
|
||||
|
||||
const raceDetector = false
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
// +build race
|
||||
|
||||
package mgo
|
||||
|
||||
const raceDetector = true
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
//+build sasl
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"github.com/globalsign/mgo/internal/sasl"
|
||||
)
|
||||
|
||||
func saslNew(cred Credential, host string) (saslStepper, error) {
|
||||
return sasl.New(cred.Username, cred.Password, cred.Mechanism, cred.Service, host)
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
//+build !sasl
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func saslNew(cred Credential, host string) (saslStepper, error) {
|
||||
return nil, fmt.Errorf("SASL support not enabled during build (-tags sasl)")
|
||||
}
|
||||
-609
@@ -1,609 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
// coarseTime is used to amortise the cost of querying the timecounter (possibly
|
||||
// incurring a syscall too) when setting a socket.lastTimeUsed value which
|
||||
// happens frequently in the hot-path.
|
||||
//
|
||||
// The lastTimeUsed value may be skewed by at least 25ms (see
|
||||
// coarseTimeProvider).
|
||||
var coarseTime *coarseTimeProvider
|
||||
|
||||
func init() {
|
||||
coarseTime = newcoarseTimeProvider(25 * time.Millisecond)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mongo server encapsulation.
|
||||
|
||||
type mongoServer struct {
|
||||
sync.RWMutex
|
||||
Addr string
|
||||
ResolvedAddr string
|
||||
tcpaddr *net.TCPAddr
|
||||
unusedSockets []*mongoSocket
|
||||
liveSockets []*mongoSocket
|
||||
sync chan bool
|
||||
dial dialer
|
||||
pingValue time.Duration
|
||||
pingIndex int
|
||||
pingWindow [6]time.Duration
|
||||
info *mongoServerInfo
|
||||
pingCount uint32
|
||||
closed bool
|
||||
abended bool
|
||||
poolWaiter *sync.Cond
|
||||
dialInfo *DialInfo
|
||||
}
|
||||
|
||||
type dialer struct {
|
||||
old func(addr net.Addr) (net.Conn, error)
|
||||
new func(addr *ServerAddr) (net.Conn, error)
|
||||
}
|
||||
|
||||
func (dial dialer) isSet() bool {
|
||||
return dial.old != nil || dial.new != nil
|
||||
}
|
||||
|
||||
type mongoServerInfo struct {
|
||||
Master bool
|
||||
Mongos bool
|
||||
Tags bson.D
|
||||
MaxWireVersion int
|
||||
SetName string
|
||||
}
|
||||
|
||||
var defaultServerInfo mongoServerInfo
|
||||
|
||||
func newServer(addr string, tcpaddr *net.TCPAddr, syncChan chan bool, dial dialer, info *DialInfo) *mongoServer {
|
||||
server := &mongoServer{
|
||||
Addr: addr,
|
||||
ResolvedAddr: tcpaddr.String(),
|
||||
tcpaddr: tcpaddr,
|
||||
sync: syncChan,
|
||||
dial: dial,
|
||||
info: &defaultServerInfo,
|
||||
pingValue: time.Hour, // Push it back before an actual ping.
|
||||
dialInfo: info,
|
||||
}
|
||||
server.poolWaiter = sync.NewCond(server)
|
||||
go server.pinger(true)
|
||||
if info.MaxIdleTimeMS != 0 {
|
||||
go server.poolShrinker()
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
var errPoolLimit = errors.New("per-server connection limit reached")
|
||||
var errPoolTimeout = errors.New("could not acquire connection within pool timeout")
|
||||
var errServerClosed = errors.New("server was closed")
|
||||
|
||||
// AcquireSocket returns a socket for communicating with the server.
|
||||
// This will attempt to reuse an old connection, if one is available. Otherwise,
|
||||
// it will establish a new one. The returned socket is owned by the call site,
|
||||
// and will return to the cache when the socket has its Release method called
|
||||
// the same number of times as AcquireSocket + Acquire were called for it.
|
||||
// If the poolLimit argument is greater than zero and the number of sockets in
|
||||
// use in this server is greater than the provided limit, errPoolLimit is
|
||||
// returned.
|
||||
func (server *mongoServer) AcquireSocket(info *DialInfo) (socket *mongoSocket, abended bool, err error) {
|
||||
return server.acquireSocketInternal(info, false)
|
||||
}
|
||||
|
||||
// AcquireSocketWithBlocking wraps AcquireSocket, but if a socket is not available, it will _not_
|
||||
// return errPoolLimit. Instead, it will block waiting for a socket to become available. If poolTimeout
|
||||
// should elapse before a socket is available, it will return errPoolTimeout.
|
||||
func (server *mongoServer) AcquireSocketWithBlocking(info *DialInfo) (socket *mongoSocket, abended bool, err error) {
|
||||
return server.acquireSocketInternal(info, true)
|
||||
}
|
||||
|
||||
func (server *mongoServer) acquireSocketInternal(info *DialInfo, shouldBlock bool) (socket *mongoSocket, abended bool, err error) {
|
||||
for {
|
||||
server.Lock()
|
||||
abended = server.abended
|
||||
if server.closed {
|
||||
server.Unlock()
|
||||
return nil, abended, errServerClosed
|
||||
}
|
||||
if info.PoolLimit > 0 {
|
||||
if shouldBlock {
|
||||
// Beautiful. Golang conditions don't have a WaitWithTimeout, so I've implemented the timeout
|
||||
// with a wait + broadcast. The broadcast will cause the loop here to re-check the timeout,
|
||||
// and fail if it is blown.
|
||||
// Yes, this is a spurious wakeup, but we can't do a directed signal without having one condition
|
||||
// variable per waiter, which would involve loop traversal in the RecycleSocket
|
||||
// method.
|
||||
// We also can't use the approach of turning a condition variable into a channel outlined in
|
||||
// https://github.com/golang/go/issues/16620, since the lock needs to be held in _this_ goroutine.
|
||||
waitDone := make(chan struct{})
|
||||
timeoutHit := false
|
||||
if info.PoolTimeout > 0 {
|
||||
go func() {
|
||||
select {
|
||||
case <-waitDone:
|
||||
case <-time.After(info.PoolTimeout):
|
||||
// timeoutHit is part of the wait condition, so needs to be changed under mutex.
|
||||
server.Lock()
|
||||
defer server.Unlock()
|
||||
timeoutHit = true
|
||||
server.poolWaiter.Broadcast()
|
||||
}
|
||||
}()
|
||||
}
|
||||
timeSpentWaiting := time.Duration(0)
|
||||
for len(server.liveSockets)-len(server.unusedSockets) >= info.PoolLimit && !timeoutHit {
|
||||
// We only count time spent in Wait(), and not time evaluating the entire loop,
|
||||
// so that in the happy non-blocking path where the condition above evaluates true
|
||||
// first time, we record a nice round zero wait time.
|
||||
waitStart := time.Now()
|
||||
// unlocks server mutex, waits, and locks again. Thus, the condition
|
||||
// above is evaluated only when the lock is held.
|
||||
server.poolWaiter.Wait()
|
||||
timeSpentWaiting += time.Since(waitStart)
|
||||
}
|
||||
close(waitDone)
|
||||
if timeoutHit {
|
||||
server.Unlock()
|
||||
stats.noticePoolTimeout(timeSpentWaiting)
|
||||
return nil, false, errPoolTimeout
|
||||
}
|
||||
// Record that we fetched a connection of of a socket list and how long we spent waiting
|
||||
stats.noticeSocketAcquisition(timeSpentWaiting)
|
||||
} else {
|
||||
if len(server.liveSockets)-len(server.unusedSockets) >= info.PoolLimit {
|
||||
server.Unlock()
|
||||
return nil, false, errPoolLimit
|
||||
}
|
||||
}
|
||||
}
|
||||
n := len(server.unusedSockets)
|
||||
if n > 0 {
|
||||
socket = server.unusedSockets[n-1]
|
||||
server.unusedSockets[n-1] = nil // Help GC.
|
||||
server.unusedSockets = server.unusedSockets[:n-1]
|
||||
serverInfo := server.info
|
||||
server.Unlock()
|
||||
err = socket.InitialAcquire(serverInfo, info)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
server.Unlock()
|
||||
socket, err = server.Connect(info)
|
||||
if err == nil {
|
||||
server.Lock()
|
||||
// We've waited for the Connect, see if we got
|
||||
// closed in the meantime
|
||||
if server.closed {
|
||||
server.Unlock()
|
||||
socket.Release()
|
||||
socket.Close()
|
||||
return nil, abended, errServerClosed
|
||||
}
|
||||
server.liveSockets = append(server.liveSockets, socket)
|
||||
server.Unlock()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Connect establishes a new connection to the server. This should
|
||||
// generally be done through server.AcquireSocket().
|
||||
func (server *mongoServer) Connect(info *DialInfo) (*mongoSocket, error) {
|
||||
server.RLock()
|
||||
master := server.info.Master
|
||||
dial := server.dial
|
||||
server.RUnlock()
|
||||
|
||||
logf("Establishing new connection to %s (timeout=%s)...", server.Addr, info.Timeout)
|
||||
var conn net.Conn
|
||||
var err error
|
||||
switch {
|
||||
case !dial.isSet():
|
||||
conn, err = net.DialTimeout("tcp", server.ResolvedAddr, info.Timeout)
|
||||
if tcpconn, ok := conn.(*net.TCPConn); ok {
|
||||
tcpconn.SetKeepAlive(true)
|
||||
} else if err == nil {
|
||||
panic("internal error: obtained TCP connection is not a *net.TCPConn!?")
|
||||
}
|
||||
case dial.old != nil:
|
||||
conn, err = dial.old(server.tcpaddr)
|
||||
case dial.new != nil:
|
||||
conn, err = dial.new(&ServerAddr{server.Addr, server.tcpaddr})
|
||||
default:
|
||||
panic("dialer is set, but both dial.old and dial.new are nil")
|
||||
}
|
||||
if err != nil {
|
||||
logf("Connection to %s failed: %v", server.Addr, err.Error())
|
||||
return nil, err
|
||||
}
|
||||
logf("Connection to %s established.", server.Addr)
|
||||
|
||||
stats.conn(+1, master)
|
||||
return newSocket(server, conn, info), nil
|
||||
}
|
||||
|
||||
// Close forces closing all sockets that are alive, whether
|
||||
// they're currently in use or not.
|
||||
func (server *mongoServer) Close() {
|
||||
server.close(false)
|
||||
}
|
||||
|
||||
// CloseIdle closing all sockets that are idle,
|
||||
// sockets currently in use will be closed after idle.
|
||||
func (server *mongoServer) CloseIdle() {
|
||||
server.close(true)
|
||||
}
|
||||
|
||||
func (server *mongoServer) close(waitForIdle bool) {
|
||||
server.Lock()
|
||||
server.closed = true
|
||||
liveSockets := server.liveSockets
|
||||
unusedSockets := server.unusedSockets
|
||||
server.liveSockets = nil
|
||||
server.unusedSockets = nil
|
||||
server.Unlock()
|
||||
logf("Connections to %s closing (%d live sockets).", server.Addr, len(liveSockets))
|
||||
for i, s := range liveSockets {
|
||||
if waitForIdle {
|
||||
s.CloseAfterIdle()
|
||||
} else {
|
||||
s.Close()
|
||||
}
|
||||
liveSockets[i] = nil
|
||||
}
|
||||
for i := range unusedSockets {
|
||||
unusedSockets[i] = nil
|
||||
}
|
||||
}
|
||||
|
||||
// RecycleSocket puts socket back into the unused cache.
|
||||
func (server *mongoServer) RecycleSocket(socket *mongoSocket) {
|
||||
server.Lock()
|
||||
if !server.closed {
|
||||
socket.lastTimeUsed = coarseTime.Now() // A rough approximation of the current time - see courseTime
|
||||
server.unusedSockets = append(server.unusedSockets, socket)
|
||||
}
|
||||
// If anybody is waiting for a connection, they should try now.
|
||||
// Note that this _has_ to be broadcast, not signal; the signature of AcquireSocket
|
||||
// and AcquireSocketWithBlocking allow the caller to specify the max number of connections,
|
||||
// rather than that being an intrinsic property of the connection pool (I assume to ensure
|
||||
// that there is always a connection available for replset topology discovery). Thus, once
|
||||
// a connection is returned to the pool, _every_ waiter needs to check if the connection count
|
||||
// is underneath their particular value for poolLimit.
|
||||
server.poolWaiter.Broadcast()
|
||||
server.Unlock()
|
||||
}
|
||||
|
||||
func removeSocket(sockets []*mongoSocket, socket *mongoSocket) []*mongoSocket {
|
||||
for i, s := range sockets {
|
||||
if s == socket {
|
||||
copy(sockets[i:], sockets[i+1:])
|
||||
n := len(sockets) - 1
|
||||
sockets[n] = nil
|
||||
sockets = sockets[:n]
|
||||
break
|
||||
}
|
||||
}
|
||||
return sockets
|
||||
}
|
||||
|
||||
// AbendSocket notifies the server that the given socket has terminated
|
||||
// abnormally, and thus should be discarded rather than cached.
|
||||
func (server *mongoServer) AbendSocket(socket *mongoSocket) {
|
||||
server.Lock()
|
||||
server.abended = true
|
||||
if server.closed {
|
||||
server.Unlock()
|
||||
return
|
||||
}
|
||||
server.liveSockets = removeSocket(server.liveSockets, socket)
|
||||
server.unusedSockets = removeSocket(server.unusedSockets, socket)
|
||||
server.Unlock()
|
||||
// Maybe just a timeout, but suggest a cluster sync up just in case.
|
||||
select {
|
||||
case server.sync <- true:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (server *mongoServer) SetInfo(info *mongoServerInfo) {
|
||||
server.Lock()
|
||||
server.info = info
|
||||
server.Unlock()
|
||||
}
|
||||
|
||||
func (server *mongoServer) Info() *mongoServerInfo {
|
||||
server.Lock()
|
||||
info := server.info
|
||||
server.Unlock()
|
||||
return info
|
||||
}
|
||||
|
||||
func (server *mongoServer) hasTags(serverTags []bson.D) bool {
|
||||
NextTagSet:
|
||||
for _, tags := range serverTags {
|
||||
NextReqTag:
|
||||
for _, req := range tags {
|
||||
for _, has := range server.info.Tags {
|
||||
if req.Name == has.Name {
|
||||
if req.Value == has.Value {
|
||||
continue NextReqTag
|
||||
}
|
||||
continue NextTagSet
|
||||
}
|
||||
}
|
||||
continue NextTagSet
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var pingDelay = 15 * time.Second
|
||||
|
||||
func (server *mongoServer) pinger(loop bool) {
|
||||
var delay time.Duration
|
||||
if raceDetector {
|
||||
// This variable is only ever touched by tests.
|
||||
globalMutex.Lock()
|
||||
delay = pingDelay
|
||||
globalMutex.Unlock()
|
||||
} else {
|
||||
delay = pingDelay
|
||||
}
|
||||
op := queryOp{
|
||||
collection: "admin.$cmd",
|
||||
query: bson.D{{Name: "ping", Value: 1}},
|
||||
flags: flagSlaveOk,
|
||||
limit: -1,
|
||||
}
|
||||
for {
|
||||
if loop {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
op := op
|
||||
|
||||
socket, _, err := server.AcquireSocket(server.dialInfo)
|
||||
if err == nil {
|
||||
start := time.Now()
|
||||
_, _ = socket.SimpleQuery(&op)
|
||||
delay := time.Since(start)
|
||||
|
||||
server.pingWindow[server.pingIndex] = delay
|
||||
server.pingIndex = (server.pingIndex + 1) % len(server.pingWindow)
|
||||
server.pingCount++
|
||||
var max time.Duration
|
||||
for i := 0; i < len(server.pingWindow) && uint32(i) < server.pingCount; i++ {
|
||||
if server.pingWindow[i] > max {
|
||||
max = server.pingWindow[i]
|
||||
}
|
||||
}
|
||||
socket.Release()
|
||||
server.Lock()
|
||||
if server.closed {
|
||||
loop = false
|
||||
}
|
||||
server.pingValue = max
|
||||
server.Unlock()
|
||||
logf("Ping for %s is %d ms", server.Addr, max/time.Millisecond)
|
||||
} else if err == errServerClosed {
|
||||
return
|
||||
}
|
||||
if !loop {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (server *mongoServer) poolShrinker() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
for _ = range ticker.C {
|
||||
if server.closed {
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
server.Lock()
|
||||
unused := len(server.unusedSockets)
|
||||
if unused < server.dialInfo.MinPoolSize {
|
||||
server.Unlock()
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
end := 0
|
||||
reclaimMap := map[*mongoSocket]struct{}{}
|
||||
// Because the acquisition and recycle are done at the tail of array,
|
||||
// the head is always the oldest unused socket.
|
||||
for _, s := range server.unusedSockets[:unused-server.dialInfo.MinPoolSize] {
|
||||
if s.lastTimeUsed.Add(time.Duration(server.dialInfo.MaxIdleTimeMS) * time.Millisecond).After(now) {
|
||||
break
|
||||
}
|
||||
end++
|
||||
reclaimMap[s] = struct{}{}
|
||||
}
|
||||
tbr := server.unusedSockets[:end]
|
||||
if end > 0 {
|
||||
next := make([]*mongoSocket, unused-end)
|
||||
copy(next, server.unusedSockets[end:])
|
||||
server.unusedSockets = next
|
||||
remainSockets := []*mongoSocket{}
|
||||
for _, s := range server.liveSockets {
|
||||
if _, ok := reclaimMap[s]; !ok {
|
||||
remainSockets = append(remainSockets, s)
|
||||
}
|
||||
}
|
||||
server.liveSockets = remainSockets
|
||||
stats.conn(-1*end, server.info.Master)
|
||||
}
|
||||
server.Unlock()
|
||||
|
||||
for _, s := range tbr {
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type mongoServerSlice []*mongoServer
|
||||
|
||||
func (s mongoServerSlice) Len() int {
|
||||
return len(s)
|
||||
}
|
||||
|
||||
func (s mongoServerSlice) Less(i, j int) bool {
|
||||
return s[i].ResolvedAddr < s[j].ResolvedAddr
|
||||
}
|
||||
|
||||
func (s mongoServerSlice) Swap(i, j int) {
|
||||
s[i], s[j] = s[j], s[i]
|
||||
}
|
||||
|
||||
func (s mongoServerSlice) Sort() {
|
||||
sort.Sort(s)
|
||||
}
|
||||
|
||||
func (s mongoServerSlice) Search(resolvedAddr string) (i int, ok bool) {
|
||||
n := len(s)
|
||||
i = sort.Search(n, func(i int) bool {
|
||||
return s[i].ResolvedAddr >= resolvedAddr
|
||||
})
|
||||
return i, i != n && s[i].ResolvedAddr == resolvedAddr
|
||||
}
|
||||
|
||||
type mongoServers struct {
|
||||
slice mongoServerSlice
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Search(resolvedAddr string) (server *mongoServer) {
|
||||
if i, ok := servers.slice.Search(resolvedAddr); ok {
|
||||
return servers.slice[i]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Add(server *mongoServer) {
|
||||
servers.slice = append(servers.slice, server)
|
||||
servers.slice.Sort()
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Remove(other *mongoServer) (server *mongoServer) {
|
||||
if i, found := servers.slice.Search(other.ResolvedAddr); found {
|
||||
server = servers.slice[i]
|
||||
copy(servers.slice[i:], servers.slice[i+1:])
|
||||
n := len(servers.slice) - 1
|
||||
servers.slice[n] = nil // Help GC.
|
||||
servers.slice = servers.slice[:n]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Slice() []*mongoServer {
|
||||
return ([]*mongoServer)(servers.slice)
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Get(i int) *mongoServer {
|
||||
return servers.slice[i]
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Len() int {
|
||||
return len(servers.slice)
|
||||
}
|
||||
|
||||
func (servers *mongoServers) Empty() bool {
|
||||
return len(servers.slice) == 0
|
||||
}
|
||||
|
||||
func (servers *mongoServers) HasMongos() bool {
|
||||
for _, s := range servers.slice {
|
||||
if s.Info().Mongos {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BestFit returns the best guess of what would be the most interesting
|
||||
// server to perform operations on at this point in time.
|
||||
func (servers *mongoServers) BestFit(mode Mode, serverTags []bson.D) *mongoServer {
|
||||
var best *mongoServer
|
||||
for _, next := range servers.slice {
|
||||
if best == nil {
|
||||
best = next
|
||||
best.RLock()
|
||||
if len(serverTags) != 0 && !next.info.Mongos && !best.hasTags(serverTags) {
|
||||
best.RUnlock()
|
||||
best = nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
next.RLock()
|
||||
swap := false
|
||||
switch {
|
||||
case len(serverTags) != 0 && !next.info.Mongos && !next.hasTags(serverTags):
|
||||
// Must have requested tags.
|
||||
case mode == Secondary && next.info.Master && !next.info.Mongos:
|
||||
// Must be a secondary or mongos.
|
||||
case next.info.Master != best.info.Master && mode != Nearest:
|
||||
// Prefer slaves, unless the mode is PrimaryPreferred.
|
||||
swap = (mode == PrimaryPreferred) != best.info.Master
|
||||
case absDuration(next.pingValue-best.pingValue) > 15*time.Millisecond:
|
||||
// Prefer nearest server.
|
||||
swap = next.pingValue < best.pingValue
|
||||
case len(next.liveSockets)-len(next.unusedSockets) < len(best.liveSockets)-len(best.unusedSockets):
|
||||
// Prefer servers with less connections.
|
||||
swap = true
|
||||
}
|
||||
if swap {
|
||||
best.RUnlock()
|
||||
best = next
|
||||
} else {
|
||||
next.RUnlock()
|
||||
}
|
||||
}
|
||||
if best != nil {
|
||||
best.RUnlock()
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func absDuration(d time.Duration) time.Duration {
|
||||
if d < 0 {
|
||||
return -d
|
||||
}
|
||||
return d
|
||||
}
|
||||
-5656
File diff suppressed because it is too large
Load Diff
-756
@@ -1,756 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo/bson"
|
||||
)
|
||||
|
||||
type replyFunc func(err error, reply *replyOp, docNum int, docData []byte)
|
||||
|
||||
type mongoSocket struct {
|
||||
sync.Mutex
|
||||
server *mongoServer // nil when cached
|
||||
conn net.Conn
|
||||
addr string // For debugging only.
|
||||
nextRequestId uint32
|
||||
replyFuncs map[uint32]replyFunc
|
||||
references int
|
||||
creds []Credential
|
||||
logout []Credential
|
||||
cachedNonce string
|
||||
gotNonce sync.Cond
|
||||
dead error
|
||||
serverInfo *mongoServerInfo
|
||||
closeAfterIdle bool
|
||||
lastTimeUsed time.Time // for time based idle socket release
|
||||
sendMeta sync.Once
|
||||
|
||||
dialInfo *DialInfo
|
||||
}
|
||||
|
||||
type queryOpFlags uint32
|
||||
|
||||
const (
|
||||
_ queryOpFlags = 1 << iota
|
||||
flagTailable
|
||||
flagSlaveOk
|
||||
flagLogReplay
|
||||
flagNoCursorTimeout
|
||||
flagAwaitData
|
||||
)
|
||||
|
||||
type queryOp struct {
|
||||
query interface{}
|
||||
collection string
|
||||
serverTags []bson.D
|
||||
selector interface{}
|
||||
replyFunc replyFunc
|
||||
mode Mode
|
||||
skip int32
|
||||
limit int32
|
||||
options queryWrapper
|
||||
hasOptions bool
|
||||
flags queryOpFlags
|
||||
readConcern string
|
||||
}
|
||||
|
||||
type queryWrapper struct {
|
||||
Query interface{} `bson:"$query"`
|
||||
OrderBy interface{} `bson:"$orderby,omitempty"`
|
||||
Hint interface{} `bson:"$hint,omitempty"`
|
||||
Explain bool `bson:"$explain,omitempty"`
|
||||
Snapshot bool `bson:"$snapshot,omitempty"`
|
||||
ReadPreference bson.D `bson:"$readPreference,omitempty"`
|
||||
MaxScan int `bson:"$maxScan,omitempty"`
|
||||
MaxTimeMS int `bson:"$maxTimeMS,omitempty"`
|
||||
Comment string `bson:"$comment,omitempty"`
|
||||
Collation *Collation `bson:"$collation,omitempty"`
|
||||
}
|
||||
|
||||
func (op *queryOp) finalQuery(socket *mongoSocket) interface{} {
|
||||
if op.flags&flagSlaveOk != 0 && socket.ServerInfo().Mongos {
|
||||
var modeName string
|
||||
switch op.mode {
|
||||
case Strong:
|
||||
modeName = "primary"
|
||||
case Monotonic, Eventual:
|
||||
modeName = "secondaryPreferred"
|
||||
case PrimaryPreferred:
|
||||
modeName = "primaryPreferred"
|
||||
case Secondary:
|
||||
modeName = "secondary"
|
||||
case SecondaryPreferred:
|
||||
modeName = "secondaryPreferred"
|
||||
case Nearest:
|
||||
modeName = "nearest"
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported read mode: %d", op.mode))
|
||||
}
|
||||
op.hasOptions = true
|
||||
op.options.ReadPreference = make(bson.D, 0, 2)
|
||||
op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "mode", Value: modeName})
|
||||
if len(op.serverTags) > 0 {
|
||||
op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "tags", Value: op.serverTags})
|
||||
}
|
||||
}
|
||||
if op.hasOptions {
|
||||
if op.query == nil {
|
||||
var empty bson.D
|
||||
op.options.Query = empty
|
||||
} else {
|
||||
op.options.Query = op.query
|
||||
}
|
||||
debugf("final query is %#v\n", &op.options)
|
||||
return &op.options
|
||||
}
|
||||
return op.query
|
||||
}
|
||||
|
||||
type getMoreOp struct {
|
||||
collection string
|
||||
limit int32
|
||||
cursorId int64
|
||||
replyFunc replyFunc
|
||||
}
|
||||
|
||||
type replyOp struct {
|
||||
flags uint32
|
||||
cursorId int64
|
||||
firstDoc int32
|
||||
replyDocs int32
|
||||
}
|
||||
|
||||
type insertOp struct {
|
||||
collection string // "database.collection"
|
||||
documents []interface{} // One or more documents to insert
|
||||
flags uint32
|
||||
}
|
||||
|
||||
type updateOp struct {
|
||||
Collection string `bson:"-"` // "database.collection"
|
||||
Selector interface{} `bson:"q"`
|
||||
Update interface{} `bson:"u"`
|
||||
Flags uint32 `bson:"-"`
|
||||
Multi bool `bson:"multi,omitempty"`
|
||||
Upsert bool `bson:"upsert,omitempty"`
|
||||
}
|
||||
|
||||
type deleteOp struct {
|
||||
Collection string `bson:"-"` // "database.collection"
|
||||
Selector interface{} `bson:"q"`
|
||||
Flags uint32 `bson:"-"`
|
||||
Limit int `bson:"limit"`
|
||||
}
|
||||
|
||||
type killCursorsOp struct {
|
||||
cursorIds []int64
|
||||
}
|
||||
|
||||
type requestInfo struct {
|
||||
bufferPos int
|
||||
replyFunc replyFunc
|
||||
}
|
||||
|
||||
func newSocket(server *mongoServer, conn net.Conn, info *DialInfo) *mongoSocket {
|
||||
socket := &mongoSocket{
|
||||
conn: conn,
|
||||
addr: server.Addr,
|
||||
server: server,
|
||||
replyFuncs: make(map[uint32]replyFunc),
|
||||
dialInfo: info,
|
||||
}
|
||||
socket.gotNonce.L = &socket.Mutex
|
||||
if err := socket.InitialAcquire(server.Info(), info); err != nil {
|
||||
panic("newSocket: InitialAcquire returned error: " + err.Error())
|
||||
}
|
||||
stats.socketsAlive(+1)
|
||||
debugf("Socket %p to %s: initialized", socket, socket.addr)
|
||||
socket.resetNonce()
|
||||
go socket.readLoop()
|
||||
return socket
|
||||
}
|
||||
|
||||
// Server returns the server that the socket is associated with.
|
||||
// It returns nil while the socket is cached in its respective server.
|
||||
func (socket *mongoSocket) Server() *mongoServer {
|
||||
socket.Lock()
|
||||
server := socket.server
|
||||
socket.Unlock()
|
||||
return server
|
||||
}
|
||||
|
||||
// ServerInfo returns details for the server at the time the socket
|
||||
// was initially acquired.
|
||||
func (socket *mongoSocket) ServerInfo() *mongoServerInfo {
|
||||
if socket == nil {
|
||||
return &mongoServerInfo{}
|
||||
}
|
||||
socket.Lock()
|
||||
serverInfo := socket.serverInfo
|
||||
socket.Unlock()
|
||||
return serverInfo
|
||||
}
|
||||
|
||||
// InitialAcquire obtains the first reference to the socket, either
|
||||
// right after the connection is made or once a recycled socket is
|
||||
// being put back in use.
|
||||
func (socket *mongoSocket) InitialAcquire(serverInfo *mongoServerInfo, dialInfo *DialInfo) error {
|
||||
socket.Lock()
|
||||
if socket.references > 0 {
|
||||
panic("Socket acquired out of cache with references")
|
||||
}
|
||||
if socket.dead != nil {
|
||||
dead := socket.dead
|
||||
socket.Unlock()
|
||||
return dead
|
||||
}
|
||||
socket.references++
|
||||
socket.serverInfo = serverInfo
|
||||
socket.dialInfo = dialInfo
|
||||
stats.socketsInUse(+1)
|
||||
stats.socketRefs(+1)
|
||||
socket.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Acquire obtains an additional reference to the socket.
|
||||
// The socket will only be recycled when it's released as many
|
||||
// times as it's been acquired.
|
||||
func (socket *mongoSocket) Acquire() (info *mongoServerInfo) {
|
||||
socket.Lock()
|
||||
if socket.references == 0 {
|
||||
panic("Socket got non-initial acquire with references == 0")
|
||||
}
|
||||
// We'll track references to dead sockets as well.
|
||||
// Caller is still supposed to release the socket.
|
||||
socket.references++
|
||||
stats.socketRefs(+1)
|
||||
serverInfo := socket.serverInfo
|
||||
socket.Unlock()
|
||||
return serverInfo
|
||||
}
|
||||
|
||||
// Release decrements a socket reference. The socket will be
|
||||
// recycled once its released as many times as it's been acquired.
|
||||
func (socket *mongoSocket) Release() {
|
||||
socket.Lock()
|
||||
if socket.references == 0 {
|
||||
panic("socket.Release() with references == 0")
|
||||
}
|
||||
socket.references--
|
||||
stats.socketRefs(-1)
|
||||
if socket.references == 0 {
|
||||
stats.socketsInUse(-1)
|
||||
server := socket.server
|
||||
closeAfterIdle := socket.closeAfterIdle
|
||||
socket.Unlock()
|
||||
socket.LogoutAll()
|
||||
if closeAfterIdle {
|
||||
socket.Close()
|
||||
} else if server != nil {
|
||||
// If the socket is dead server is nil.
|
||||
server.RecycleSocket(socket)
|
||||
}
|
||||
} else {
|
||||
socket.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// SetTimeout changes the timeout used on socket operations.
|
||||
func (socket *mongoSocket) SetTimeout(d time.Duration) {
|
||||
socket.Lock()
|
||||
socket.dialInfo.ReadTimeout = d
|
||||
socket.dialInfo.WriteTimeout = d
|
||||
socket.Unlock()
|
||||
}
|
||||
|
||||
type deadlineType int
|
||||
|
||||
const (
|
||||
readDeadline deadlineType = 1
|
||||
writeDeadline deadlineType = 2
|
||||
)
|
||||
|
||||
func (socket *mongoSocket) updateDeadline(which deadlineType) {
|
||||
var when time.Time
|
||||
var whichStr string
|
||||
switch which {
|
||||
case readDeadline | writeDeadline:
|
||||
if socket.dialInfo.roundTripTimeout() == 0 {
|
||||
return
|
||||
}
|
||||
whichStr = "read/write"
|
||||
when = time.Now().Add(socket.dialInfo.roundTripTimeout())
|
||||
socket.conn.SetDeadline(when)
|
||||
|
||||
case readDeadline:
|
||||
if socket.dialInfo.ReadTimeout == zeroDuration {
|
||||
return
|
||||
}
|
||||
whichStr = "read"
|
||||
when = time.Now().Add(socket.dialInfo.ReadTimeout)
|
||||
socket.conn.SetReadDeadline(when)
|
||||
|
||||
case writeDeadline:
|
||||
if socket.dialInfo.WriteTimeout == zeroDuration {
|
||||
return
|
||||
}
|
||||
whichStr = "write"
|
||||
when = time.Now().Add(socket.dialInfo.WriteTimeout)
|
||||
socket.conn.SetWriteDeadline(when)
|
||||
|
||||
default:
|
||||
panic("invalid parameter to updateDeadline")
|
||||
}
|
||||
|
||||
debugf("Socket %p to %s: updated %s deadline to %s", socket, socket.addr, whichStr, when)
|
||||
}
|
||||
|
||||
// Close terminates the socket use.
|
||||
func (socket *mongoSocket) Close() {
|
||||
socket.kill(errors.New("Closed explicitly"), false)
|
||||
}
|
||||
|
||||
// CloseAfterIdle terminates an idle socket, which has a zero
|
||||
// reference, or marks the socket to be terminate after idle.
|
||||
func (socket *mongoSocket) CloseAfterIdle() {
|
||||
socket.Lock()
|
||||
if socket.references == 0 {
|
||||
socket.Unlock()
|
||||
socket.Close()
|
||||
logf("Socket %p to %s: idle and close.", socket, socket.addr)
|
||||
return
|
||||
}
|
||||
socket.closeAfterIdle = true
|
||||
socket.Unlock()
|
||||
logf("Socket %p to %s: close after idle.", socket, socket.addr)
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) kill(err error, abend bool) {
|
||||
socket.Lock()
|
||||
if socket.dead != nil {
|
||||
debugf("Socket %p to %s: killed again: %s (previously: %s)", socket, socket.addr, err.Error(), socket.dead.Error())
|
||||
socket.Unlock()
|
||||
return
|
||||
}
|
||||
logf("Socket %p to %s: closing: %s (abend=%v)", socket, socket.addr, err.Error(), abend)
|
||||
socket.dead = err
|
||||
socket.conn.Close()
|
||||
stats.socketsAlive(-1)
|
||||
replyFuncs := socket.replyFuncs
|
||||
socket.replyFuncs = make(map[uint32]replyFunc)
|
||||
server := socket.server
|
||||
socket.server = nil
|
||||
socket.gotNonce.Broadcast()
|
||||
socket.Unlock()
|
||||
for _, replyFunc := range replyFuncs {
|
||||
logf("Socket %p to %s: notifying replyFunc of closed socket: %s", socket, socket.addr, err.Error())
|
||||
replyFunc(err, nil, -1, nil)
|
||||
}
|
||||
if abend {
|
||||
server.AbendSocket(socket)
|
||||
}
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) SimpleQuery(op *queryOp) (data []byte, err error) {
|
||||
var wait, change sync.Mutex
|
||||
var replyDone bool
|
||||
var replyData []byte
|
||||
var replyErr error
|
||||
wait.Lock()
|
||||
op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) {
|
||||
change.Lock()
|
||||
if !replyDone {
|
||||
replyDone = true
|
||||
replyErr = err
|
||||
if err == nil {
|
||||
replyData = docData
|
||||
}
|
||||
}
|
||||
change.Unlock()
|
||||
wait.Unlock()
|
||||
}
|
||||
err = socket.Query(op)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wait.Lock()
|
||||
change.Lock()
|
||||
data = replyData
|
||||
err = replyErr
|
||||
change.Unlock()
|
||||
return data, err
|
||||
}
|
||||
|
||||
var bytesBufferPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, 0, 256)
|
||||
},
|
||||
}
|
||||
|
||||
func (socket *mongoSocket) Query(ops ...interface{}) (err error) {
|
||||
|
||||
if lops := socket.flushLogout(); len(lops) > 0 {
|
||||
ops = append(lops, ops...)
|
||||
}
|
||||
|
||||
buf := bytesBufferPool.Get().([]byte)
|
||||
defer func() {
|
||||
bytesBufferPool.Put(buf[:0])
|
||||
}()
|
||||
|
||||
// Serialize operations synchronously to avoid interrupting
|
||||
// other goroutines while we can't really be sending data.
|
||||
// Also, record id positions so that we can compute request
|
||||
// ids at once later with the lock already held.
|
||||
requests := make([]requestInfo, len(ops))
|
||||
requestCount := 0
|
||||
|
||||
for _, op := range ops {
|
||||
debugf("Socket %p to %s: serializing op: %#v", socket, socket.addr, op)
|
||||
if qop, ok := op.(*queryOp); ok {
|
||||
if cmd, ok := qop.query.(*findCmd); ok {
|
||||
debugf("Socket %p to %s: find command: %#v", socket, socket.addr, cmd)
|
||||
}
|
||||
}
|
||||
start := len(buf)
|
||||
var replyFunc replyFunc
|
||||
switch op := op.(type) {
|
||||
|
||||
case *updateOp:
|
||||
buf = addHeader(buf, 2001)
|
||||
buf = addInt32(buf, 0) // Reserved
|
||||
buf = addCString(buf, op.Collection)
|
||||
buf = addInt32(buf, int32(op.Flags))
|
||||
debugf("Socket %p to %s: serializing selector document: %#v", socket, socket.addr, op.Selector)
|
||||
buf, err = addBSON(buf, op.Selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
debugf("Socket %p to %s: serializing update document: %#v", socket, socket.addr, op.Update)
|
||||
buf, err = addBSON(buf, op.Update)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case *insertOp:
|
||||
buf = addHeader(buf, 2002)
|
||||
buf = addInt32(buf, int32(op.flags))
|
||||
buf = addCString(buf, op.collection)
|
||||
for _, doc := range op.documents {
|
||||
debugf("Socket %p to %s: serializing document for insertion: %#v", socket, socket.addr, doc)
|
||||
buf, err = addBSON(buf, doc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
case *queryOp:
|
||||
buf = addHeader(buf, 2004)
|
||||
buf = addInt32(buf, int32(op.flags))
|
||||
buf = addCString(buf, op.collection)
|
||||
buf = addInt32(buf, op.skip)
|
||||
buf = addInt32(buf, op.limit)
|
||||
buf, err = addBSON(buf, op.finalQuery(socket))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if op.selector != nil {
|
||||
buf, err = addBSON(buf, op.selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
replyFunc = op.replyFunc
|
||||
|
||||
case *getMoreOp:
|
||||
buf = addHeader(buf, 2005)
|
||||
buf = addInt32(buf, 0) // Reserved
|
||||
buf = addCString(buf, op.collection)
|
||||
buf = addInt32(buf, op.limit)
|
||||
buf = addInt64(buf, op.cursorId)
|
||||
replyFunc = op.replyFunc
|
||||
|
||||
case *deleteOp:
|
||||
buf = addHeader(buf, 2006)
|
||||
buf = addInt32(buf, 0) // Reserved
|
||||
buf = addCString(buf, op.Collection)
|
||||
buf = addInt32(buf, int32(op.Flags))
|
||||
debugf("Socket %p to %s: serializing selector document: %#v", socket, socket.addr, op.Selector)
|
||||
buf, err = addBSON(buf, op.Selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case *killCursorsOp:
|
||||
buf = addHeader(buf, 2007)
|
||||
buf = addInt32(buf, 0) // Reserved
|
||||
buf = addInt32(buf, int32(len(op.cursorIds)))
|
||||
for _, cursorId := range op.cursorIds {
|
||||
buf = addInt64(buf, cursorId)
|
||||
}
|
||||
|
||||
default:
|
||||
panic("internal error: unknown operation type")
|
||||
}
|
||||
|
||||
setInt32(buf, start, int32(len(buf)-start))
|
||||
|
||||
if replyFunc != nil {
|
||||
request := &requests[requestCount]
|
||||
request.replyFunc = replyFunc
|
||||
request.bufferPos = start
|
||||
requestCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Buffer is ready for the pipe. Lock, allocate ids, and enqueue.
|
||||
|
||||
socket.Lock()
|
||||
if socket.dead != nil {
|
||||
dead := socket.dead
|
||||
socket.Unlock()
|
||||
debugf("Socket %p to %s: failing query, already closed: %s", socket, socket.addr, socket.dead.Error())
|
||||
// XXX This seems necessary in case the session is closed concurrently
|
||||
// with a query being performed, but it's not yet tested:
|
||||
for i := 0; i != requestCount; i++ {
|
||||
request := &requests[i]
|
||||
if request.replyFunc != nil {
|
||||
request.replyFunc(dead, nil, -1, nil)
|
||||
}
|
||||
}
|
||||
return dead
|
||||
}
|
||||
|
||||
wasWaiting := len(socket.replyFuncs) > 0
|
||||
|
||||
// Reserve id 0 for requests which should have no responses.
|
||||
requestId := socket.nextRequestId + 1
|
||||
if requestId == 0 {
|
||||
requestId++
|
||||
}
|
||||
socket.nextRequestId = requestId + uint32(requestCount)
|
||||
for i := 0; i != requestCount; i++ {
|
||||
request := &requests[i]
|
||||
setInt32(buf, request.bufferPos+4, int32(requestId))
|
||||
socket.replyFuncs[requestId] = request.replyFunc
|
||||
requestId++
|
||||
}
|
||||
socket.Unlock()
|
||||
debugf("Socket %p to %s: sending %d op(s) (%d bytes)", socket, socket.addr, len(ops), len(buf))
|
||||
|
||||
stats.sentOps(len(ops))
|
||||
socket.updateDeadline(writeDeadline)
|
||||
_, err = socket.conn.Write(buf)
|
||||
if !wasWaiting && requestCount > 0 {
|
||||
socket.updateDeadline(readDeadline)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func fill(r net.Conn, b []byte) error {
|
||||
l := len(b)
|
||||
n, err := r.Read(b)
|
||||
for n != l && err == nil {
|
||||
var ni int
|
||||
ni, err = r.Read(b[n:])
|
||||
n += ni
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Estimated minimum cost per socket: 1 goroutine + memory for the largest
|
||||
// document ever seen.
|
||||
func (socket *mongoSocket) readLoop() {
|
||||
p := make([]byte, 36) // 16 from header + 20 from OP_REPLY fixed fields
|
||||
s := make([]byte, 4)
|
||||
conn := socket.conn // No locking, conn never changes.
|
||||
for {
|
||||
err := fill(conn, p)
|
||||
if err != nil {
|
||||
socket.kill(err, true)
|
||||
return
|
||||
}
|
||||
|
||||
totalLen := getInt32(p, 0)
|
||||
responseTo := getInt32(p, 8)
|
||||
opCode := getInt32(p, 12)
|
||||
|
||||
// Don't use socket.server.Addr here. socket is not
|
||||
// locked and socket.server may go away.
|
||||
debugf("Socket %p to %s: got reply (%d bytes)", socket, socket.addr, totalLen)
|
||||
|
||||
_ = totalLen
|
||||
|
||||
if opCode != 1 {
|
||||
socket.kill(errors.New("opcode != 1, corrupted data?"), true)
|
||||
return
|
||||
}
|
||||
|
||||
reply := replyOp{
|
||||
flags: uint32(getInt32(p, 16)),
|
||||
cursorId: getInt64(p, 20),
|
||||
firstDoc: getInt32(p, 28),
|
||||
replyDocs: getInt32(p, 32),
|
||||
}
|
||||
|
||||
stats.receivedOps(+1)
|
||||
stats.receivedDocs(int(reply.replyDocs))
|
||||
|
||||
socket.Lock()
|
||||
replyFunc, ok := socket.replyFuncs[uint32(responseTo)]
|
||||
if ok {
|
||||
delete(socket.replyFuncs, uint32(responseTo))
|
||||
}
|
||||
socket.Unlock()
|
||||
|
||||
if replyFunc != nil && reply.replyDocs == 0 {
|
||||
replyFunc(nil, &reply, -1, nil)
|
||||
} else {
|
||||
for i := 0; i != int(reply.replyDocs); i++ {
|
||||
err := fill(conn, s)
|
||||
if err != nil {
|
||||
if replyFunc != nil {
|
||||
replyFunc(err, nil, -1, nil)
|
||||
}
|
||||
socket.kill(err, true)
|
||||
return
|
||||
}
|
||||
|
||||
b := make([]byte, int(getInt32(s, 0)))
|
||||
|
||||
// copy(b, s) in an efficient way.
|
||||
b[0] = s[0]
|
||||
b[1] = s[1]
|
||||
b[2] = s[2]
|
||||
b[3] = s[3]
|
||||
|
||||
err = fill(conn, b[4:])
|
||||
if err != nil {
|
||||
if replyFunc != nil {
|
||||
replyFunc(err, nil, -1, nil)
|
||||
}
|
||||
socket.kill(err, true)
|
||||
return
|
||||
}
|
||||
|
||||
if globalDebug && globalLogger != nil {
|
||||
m := bson.M{}
|
||||
if err := bson.Unmarshal(b, m); err == nil {
|
||||
debugf("Socket %p to %s: received document: %#v", socket, socket.addr, m)
|
||||
}
|
||||
}
|
||||
|
||||
if replyFunc != nil {
|
||||
replyFunc(nil, &reply, i, b)
|
||||
}
|
||||
|
||||
// XXX Do bound checking against totalLen.
|
||||
}
|
||||
}
|
||||
|
||||
socket.Lock()
|
||||
if len(socket.replyFuncs) == 0 {
|
||||
// Nothing else to read for now. Disable deadline.
|
||||
socket.conn.SetReadDeadline(time.Time{})
|
||||
} else {
|
||||
socket.updateDeadline(readDeadline)
|
||||
}
|
||||
socket.Unlock()
|
||||
|
||||
// XXX Do bound checking against totalLen.
|
||||
}
|
||||
}
|
||||
|
||||
var emptyHeader = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
|
||||
|
||||
func addHeader(b []byte, opcode int) []byte {
|
||||
i := len(b)
|
||||
b = append(b, emptyHeader...)
|
||||
// Enough for current opcodes.
|
||||
b[i+12] = byte(opcode)
|
||||
b[i+13] = byte(opcode >> 8)
|
||||
return b
|
||||
}
|
||||
|
||||
func addInt32(b []byte, i int32) []byte {
|
||||
return append(b, byte(i), byte(i>>8), byte(i>>16), byte(i>>24))
|
||||
}
|
||||
|
||||
func addInt64(b []byte, i int64) []byte {
|
||||
return append(b, byte(i), byte(i>>8), byte(i>>16), byte(i>>24),
|
||||
byte(i>>32), byte(i>>40), byte(i>>48), byte(i>>56))
|
||||
}
|
||||
|
||||
func addCString(b []byte, s string) []byte {
|
||||
b = append(b, []byte(s)...)
|
||||
b = append(b, 0)
|
||||
return b
|
||||
}
|
||||
|
||||
func addBSON(b []byte, doc interface{}) ([]byte, error) {
|
||||
if doc == nil {
|
||||
return append(b, 5, 0, 0, 0, 0), nil
|
||||
}
|
||||
data, err := bson.MarshalBuffer(doc, b)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func setInt32(b []byte, pos int, i int32) {
|
||||
b[pos] = byte(i)
|
||||
b[pos+1] = byte(i >> 8)
|
||||
b[pos+2] = byte(i >> 16)
|
||||
b[pos+3] = byte(i >> 24)
|
||||
}
|
||||
|
||||
func getInt32(b []byte, pos int) int32 {
|
||||
return (int32(b[pos+0])) |
|
||||
(int32(b[pos+1]) << 8) |
|
||||
(int32(b[pos+2]) << 16) |
|
||||
(int32(b[pos+3]) << 24)
|
||||
}
|
||||
|
||||
func getInt64(b []byte, pos int) int64 {
|
||||
return (int64(b[pos+0])) |
|
||||
(int64(b[pos+1]) << 8) |
|
||||
(int64(b[pos+2]) << 16) |
|
||||
(int64(b[pos+3]) << 24) |
|
||||
(int64(b[pos+4]) << 32) |
|
||||
(int64(b[pos+5]) << 40) |
|
||||
(int64(b[pos+6]) << 48) |
|
||||
(int64(b[pos+7]) << 56)
|
||||
}
|
||||
-184
@@ -1,184 +0,0 @@
|
||||
// mgo - MongoDB driver for Go
|
||||
//
|
||||
// Copyright (c) 2010-2012 - Gustavo Niemeyer <gustavo@niemeyer.net>
|
||||
//
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice, this
|
||||
// list of conditions and the following disclaimer.
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package mgo
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var stats *Stats
|
||||
var statsMutex sync.Mutex
|
||||
|
||||
// SetStats enable database state monitoring
|
||||
func SetStats(enabled bool) {
|
||||
statsMutex.Lock()
|
||||
if enabled {
|
||||
if stats == nil {
|
||||
stats = &Stats{}
|
||||
}
|
||||
} else {
|
||||
stats = nil
|
||||
}
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
|
||||
// GetStats return the current database state
|
||||
func GetStats() (snapshot Stats) {
|
||||
statsMutex.Lock()
|
||||
snapshot = *stats
|
||||
statsMutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// ResetStats reset Stats to the previous database state
|
||||
func ResetStats() {
|
||||
statsMutex.Lock()
|
||||
debug("Resetting stats")
|
||||
old := stats
|
||||
stats = &Stats{}
|
||||
// These are absolute values:
|
||||
stats.Clusters = old.Clusters
|
||||
stats.SocketsInUse = old.SocketsInUse
|
||||
stats.SocketsAlive = old.SocketsAlive
|
||||
stats.SocketRefs = old.SocketRefs
|
||||
statsMutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Stats holds info on the database state
|
||||
//
|
||||
// Relevant documentation:
|
||||
//
|
||||
// https://docs.mongodb.com/manual/reference/command/serverStatus/
|
||||
//
|
||||
// TODO outdated fields ?
|
||||
type Stats struct {
|
||||
Clusters int
|
||||
MasterConns int
|
||||
SlaveConns int
|
||||
SentOps int
|
||||
ReceivedOps int
|
||||
ReceivedDocs int
|
||||
SocketsAlive int
|
||||
SocketsInUse int
|
||||
SocketRefs int
|
||||
TimesSocketAcquired int
|
||||
TimesWaitedForPool int
|
||||
TotalPoolWaitTime time.Duration
|
||||
PoolTimeouts int
|
||||
}
|
||||
|
||||
func (stats *Stats) cluster(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.Clusters += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) conn(delta int, master bool) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
if master {
|
||||
stats.MasterConns += delta
|
||||
} else {
|
||||
stats.SlaveConns += delta
|
||||
}
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) sentOps(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.SentOps += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) receivedOps(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.ReceivedOps += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) receivedDocs(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.ReceivedDocs += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) socketsInUse(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.SocketsInUse += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) socketsAlive(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.SocketsAlive += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) socketRefs(delta int) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.SocketRefs += delta
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) noticeSocketAcquisition(waitTime time.Duration) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.TimesSocketAcquired++
|
||||
stats.TotalPoolWaitTime += waitTime
|
||||
if waitTime > 0 {
|
||||
stats.TimesWaitedForPool++
|
||||
}
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (stats *Stats) noticePoolTimeout(waitTime time.Duration) {
|
||||
if stats != nil {
|
||||
statsMutex.Lock()
|
||||
stats.TimesWaitedForPool++
|
||||
stats.PoolTimeouts++
|
||||
stats.TotalPoolWaitTime += waitTime
|
||||
statsMutex.Unlock()
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -69,4 +69,4 @@ issues:
|
||||
exclude-use-default: false
|
||||
|
||||
service:
|
||||
golangci-lint-version: 1.15.x
|
||||
golangci-lint-version: 1.19.x
|
||||
|
||||
+5
-5
@@ -4,21 +4,21 @@ services:
|
||||
- mongodb
|
||||
|
||||
go:
|
||||
- "1.12.x"
|
||||
- "1.13.x"
|
||||
|
||||
install: true
|
||||
|
||||
before_install:
|
||||
- export TZ=America/Chicago
|
||||
- curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.17.1
|
||||
- curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.19.1
|
||||
- golangci-lint --version
|
||||
- go get github.com/mattn/goveralls
|
||||
- export MONGO_TEST=mongodb://127.0.0.1:27017
|
||||
- export PATH=$(pwd)/bin:$PATH
|
||||
|
||||
script:
|
||||
- GO111MODULE=on go get ./...
|
||||
- GO111MODULE=on go mod vendor
|
||||
- GO111MODULE=on go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
|
||||
- go get ./...
|
||||
- go mod vendor
|
||||
- go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
|
||||
- golangci-lint run --tests=false || travis_terminate 1;
|
||||
- $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 Umputun
|
||||
Copyright (c) 2019 Umputun
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
+1
-12
@@ -2,8 +2,6 @@ package avatar
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -58,7 +56,7 @@ func (b *BoltDB) Put(userID string, reader io.Reader) (avatar string, err error)
|
||||
return errors.Wrapf(err, "can't put to bucket with %s", avatarID)
|
||||
}
|
||||
// store sha1 of the image
|
||||
return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(b.sha1(buf.Bytes(), avatarID)))
|
||||
return tx.Bucket([]byte(metasBktName)).Put([]byte(avatarID), []byte(hash(buf.Bytes(), avatarID)))
|
||||
})
|
||||
return avatarID, err
|
||||
}
|
||||
@@ -130,12 +128,3 @@ func (b *BoltDB) Close() error {
|
||||
func (b *BoltDB) String() string {
|
||||
return fmt.Sprintf("boltdb, path=%s", b.fileName)
|
||||
}
|
||||
|
||||
func (b *BoltDB) sha1(data []byte, avatarID string) (id string) {
|
||||
h := sha1.New()
|
||||
if _, err := h.Write(data); err != nil {
|
||||
log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
|
||||
return encodeID(avatarID)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
+97
-66
@@ -2,122 +2,153 @@ package avatar
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo"
|
||||
"github.com/go-pkgz/mongo"
|
||||
"github.com/pkg/errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/gridfs"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// NewGridFS makes gridfs (mongo) avatar store
|
||||
func NewGridFS(conn *mongo.Connection) *GridFS {
|
||||
return &GridFS{Connection: conn}
|
||||
func NewGridFS(client *mongo.Client, dbName, bucketName string, timeout time.Duration) *GridFS {
|
||||
return &GridFS{client: client, db: client.Database(dbName), bucketName: bucketName, timeout: timeout}
|
||||
}
|
||||
|
||||
// GridFS implements Store for GridFS
|
||||
type GridFS struct {
|
||||
Connection *mongo.Connection
|
||||
client *mongo.Client
|
||||
db *mongo.Database
|
||||
bucketName string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// Put avatar to gridfs object, try to resize
|
||||
func (gf *GridFS) Put(userID string, reader io.Reader) (avatar string, err error) {
|
||||
id := encodeID(userID)
|
||||
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
|
||||
fh, e := dbase.GridFS("fs").Create(id + imgSfx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer func() {
|
||||
if err = fh.Close(); err != nil {
|
||||
log.Printf("[WARN] can't close avatar file %v, %s", fh, err)
|
||||
}
|
||||
}()
|
||||
bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
_, e = io.Copy(fh, reader)
|
||||
return e
|
||||
})
|
||||
buf := &bytes.Buffer{}
|
||||
if _, err = io.Copy(buf, reader); err != nil {
|
||||
return "", errors.Wrapf(err, "can't read avatar for %s", userID)
|
||||
}
|
||||
|
||||
avaHash := hash(buf.Bytes(), id)
|
||||
_, err = bucket.UploadFromStream(id+imgSfx, buf, &options.UploadOptions{Metadata: bson.M{"hash": avaHash}})
|
||||
return id + imgSfx, err
|
||||
}
|
||||
|
||||
// Get avatar reader for avatar id.image
|
||||
func (gf *GridFS) Get(avatar string) (reader io.ReadCloser, size int, err error) {
|
||||
bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
buf := &bytes.Buffer{}
|
||||
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
|
||||
fh, e := dbase.GridFS("fs").Open(avatar)
|
||||
if e != nil {
|
||||
return errors.Wrapf(e, "can't load avatar %s", avatar)
|
||||
}
|
||||
if _, e = io.Copy(buf, fh); e != nil {
|
||||
return errors.Wrapf(e, "can't copy avatar %s", avatar)
|
||||
}
|
||||
size = int(fh.Size())
|
||||
return fh.Close()
|
||||
})
|
||||
return ioutil.NopCloser(buf), size, err
|
||||
sz, e := bucket.DownloadToStreamByName(avatar, buf)
|
||||
return ioutil.NopCloser(buf), int(sz), errors.Wrapf(e, "can't read avatar %s", avatar)
|
||||
}
|
||||
|
||||
//
|
||||
// ID returns a fingerprint of the avatar content. Uses MD5 because gridfs provides it directly
|
||||
func (gf *GridFS) ID(avatar string) (id string) {
|
||||
err := gf.Connection.WithDB(func(dbase *mgo.Database) error {
|
||||
fh, e := dbase.GridFS("fs").Open(avatar)
|
||||
if e != nil {
|
||||
return errors.Wrapf(e, "can't open avatar %s", avatar)
|
||||
}
|
||||
id = fh.MD5()
|
||||
return errors.Wrapf(fh.Close(), "can't close avatar")
|
||||
})
|
||||
|
||||
finfo := struct {
|
||||
ID primitive.ObjectID `bson:"_id"`
|
||||
Len int `bson:"length"`
|
||||
FileName string `bson:"filename"`
|
||||
MetaData struct {
|
||||
Hash string `bson:"hash"`
|
||||
} `bson:"metadata"`
|
||||
}{}
|
||||
|
||||
bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
|
||||
if err != nil {
|
||||
log.Printf("[DEBUG] can't get file info '%s', %s", avatar, err)
|
||||
return encodeID(avatar)
|
||||
}
|
||||
return id
|
||||
cursor, err := bucket.Find(bson.M{"filename": avatar})
|
||||
if err != nil {
|
||||
return encodeID(avatar)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
if found := cursor.Next(ctx); found {
|
||||
if err = cursor.Decode(&finfo); err != nil {
|
||||
return encodeID(avatar)
|
||||
}
|
||||
return finfo.MetaData.Hash
|
||||
}
|
||||
return encodeID(avatar)
|
||||
}
|
||||
|
||||
// Remove avatar from gridfs
|
||||
func (gf *GridFS) Remove(avatar string) error {
|
||||
return gf.Connection.WithDB(func(dbase *mgo.Database) error {
|
||||
fh, e := dbase.GridFS("fs").Open(avatar)
|
||||
if e != nil {
|
||||
return errors.Wrapf(e, "can't get avatar %s", avatar)
|
||||
bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cursor, err := bucket.Find(bson.M{"filename": avatar})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r := struct {
|
||||
ID primitive.ObjectID `bson:"_id"`
|
||||
}{}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
if found := cursor.Next(ctx); found {
|
||||
if err := cursor.Decode(&r); err != nil {
|
||||
return err
|
||||
}
|
||||
if e = fh.Close(); e != nil {
|
||||
log.Printf("[WARN] can't close avatar %s, %s", avatar, e)
|
||||
}
|
||||
return dbase.GridFS("fs").Remove(avatar)
|
||||
})
|
||||
return bucket.Delete(r.ID)
|
||||
}
|
||||
return errors.Errorf("avatar %s not found", avatar)
|
||||
}
|
||||
|
||||
// List all avatars (ids) on gfs
|
||||
// note: id includes .image suffix
|
||||
func (gf *GridFS) List() (ids []string, err error) {
|
||||
|
||||
type gfsFile struct {
|
||||
UploadDate time.Time `bson:"uploadDate"`
|
||||
Length int64 `bson:",minsize"`
|
||||
MD5 string
|
||||
Filename string `bson:",omitempty"`
|
||||
bucket, err := gridfs.NewBucket(gf.db, &options.BucketOptions{Name: &gf.bucketName})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files := []gfsFile{}
|
||||
err = gf.Connection.WithDB(func(dbase *mgo.Database) error {
|
||||
return dbase.GridFS("fs").Find(nil).All(&files)
|
||||
})
|
||||
|
||||
for _, f := range files {
|
||||
ids = append(ids, f.Filename)
|
||||
gfsFile := struct {
|
||||
Filename string `bson:"filename,omitempty"`
|
||||
}{}
|
||||
cursor, err := bucket.Find(bson.M{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ids, errors.Wrap(err, "can't list avatars")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
for cursor.Next(ctx) {
|
||||
if err := cursor.Decode(&gfsFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, gfsFile.Filename)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// Close gridfs does nothing but satisfies interface
|
||||
func (gf *GridFS) Close() error {
|
||||
return nil
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gf.timeout)
|
||||
defer cancel()
|
||||
return gf.client.Disconnect(ctx)
|
||||
}
|
||||
|
||||
func (gf *GridFS) String() string {
|
||||
return fmt.Sprintf("mongo (grid fs), conn=%s", gf.Connection)
|
||||
return fmt.Sprintf("mongo (grid fs), db=%s, bucket=%s", gf.db.Name(), gf.bucketName)
|
||||
}
|
||||
|
||||
+25
-5
@@ -3,7 +3,9 @@ package avatar
|
||||
//go:generate sh -c "mockery -inpkg -name Store -print > /tmp/mock.tmp && mv /tmp/mock.tmp store_mock.go"
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
_ "image/gif" // initializing packages for supporting GIF
|
||||
_ "image/jpeg" // initializing packages for supporting JPEG.
|
||||
@@ -17,8 +19,9 @@ import (
|
||||
|
||||
bolt "github.com/coreos/bbolt"
|
||||
"github.com/go-pkgz/auth/token"
|
||||
"github.com/go-pkgz/mongo"
|
||||
"github.com/pkg/errors"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// imgSfx for avatars
|
||||
@@ -46,15 +49,23 @@ func NewStore(uri string) (Store, error) {
|
||||
case !strings.Contains(uri, "://"):
|
||||
return NewLocalFS(uri), nil
|
||||
case strings.HasPrefix(uri, "mongodb://"):
|
||||
db, coll, u, err := parseExtMongoURI(uri)
|
||||
|
||||
db, bucketName, u, err := parseExtMongoURI(uri)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "can't parse mongo store uri %s", uri)
|
||||
}
|
||||
mg, err := mongo.NewServerWithURL(u, time.Second)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(u))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make mongo server")
|
||||
return nil, errors.Wrap(err, "failed to connect to mongo server")
|
||||
}
|
||||
return NewGridFS(mongo.NewConnection(mg, db, coll)), nil
|
||||
if err = client.Ping(ctx, nil); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to connect to mongo server")
|
||||
}
|
||||
return NewGridFS(client, db, bucketName, time.Second*5), nil
|
||||
case strings.HasPrefix(uri, "bolt://"):
|
||||
return NewBoltDB(strings.TrimPrefix(uri, "bolt://"), bolt.Options{})
|
||||
}
|
||||
@@ -113,3 +124,12 @@ func parseExtMongoURI(uri string) (db, collection, cleanURI string, err error) {
|
||||
u.RawQuery = q.Encode()
|
||||
return db, collection, u.String(), nil
|
||||
}
|
||||
|
||||
func hash(data []byte, avatarID string) (id string) {
|
||||
h := sha1.New()
|
||||
if _, err := h.Write(data); err != nil {
|
||||
log.Printf("[DEBUG] can't apply sha1 for content of '%s', %s", avatarID, err)
|
||||
return encodeID(avatarID)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
+8
-4
@@ -5,21 +5,25 @@ require (
|
||||
github.com/coreos/bbolt v1.3.3
|
||||
github.com/dghubble/oauth1 v0.6.0
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
|
||||
github.com/go-chi/chi v4.0.2+incompatible // indirect
|
||||
github.com/go-pkgz/auth/_example v0.0.0-20190722170031-705d3f732438 // indirect
|
||||
github.com/go-pkgz/mongo v1.1.2
|
||||
github.com/go-pkgz/rest v1.4.1
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.2
|
||||
github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/stretchr/testify v1.3.0
|
||||
github.com/tidwall/pretty v1.0.0 // indirect
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
|
||||
github.com/xdg/stringprep v1.0.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.3 // indirect
|
||||
go.mongodb.org/mongo-driver v1.1.1
|
||||
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
|
||||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect
|
||||
google.golang.org/appengine v1.6.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/oauth2.v3 v3.10.1
|
||||
)
|
||||
|
||||
|
||||
+28
-23
@@ -7,9 +7,7 @@ cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlr
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM=
|
||||
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
|
||||
github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -21,30 +19,17 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313 h1:GSPjYG49Uqn3S1oeFgJtlGI3ykTavl/yvYgZlz6wsoI=
|
||||
github.com/gavv/httpexpect v0.0.0-20180803094507-bdde30871313/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
|
||||
github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d h1:oYXrtNhqNKL1dVtKdv8XUq5zqdGVFNQ0/4tvccXZOLM=
|
||||
github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
|
||||
github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/go-chi/chi v4.0.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs=
|
||||
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/go-pkgz/auth v0.4.1/go.mod h1:CWtB8dHmOv+TfF3MUzKwk/YwTLepC2TaDL05A+pFVBM=
|
||||
github.com/go-pkgz/auth/_example v0.0.0-20190722170031-705d3f732438 h1:A9QL16LR1e+WzK85qUX1tIVe+9dhuOZrnp/5Lsyl0Po=
|
||||
github.com/go-pkgz/auth/_example v0.0.0-20190722170031-705d3f732438/go.mod h1:rvZtFFkmm3p+E0CHmfUqTGKweCVg2ddsRUrEEPB1iaE=
|
||||
github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
|
||||
github.com/go-pkgz/lgr v0.6.2 h1:Twf2YIe2J5tg7mKs+IkDDxrDF7GWlTCl/LzqELWjT5o=
|
||||
github.com/go-pkgz/lgr v0.6.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
|
||||
github.com/go-pkgz/mongo v1.0.0/go.mod h1:R9si/F2aJsjz4MUxhzuppIHY8yLV3YCeuCpgcI50cu4=
|
||||
github.com/go-pkgz/mongo v1.1.2 h1:2Vqn3CWQJkkx4gxxDiQUitAW2FN/CH26lKHkipmpKcc=
|
||||
github.com/go-pkgz/mongo v1.1.2/go.mod h1:0NkWnzpiUxoL5fYZuttCtJrpC67oNDidfYxcdPqHTf0=
|
||||
github.com/go-pkgz/rest v1.2.0/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
|
||||
github.com/go-pkgz/rest v1.4.1 h1:DmaVLPH2O7yLehrWOW0uz01d2mVHz9fBR/iuTiPRzaw=
|
||||
github.com/go-pkgz/rest v1.4.1/go.mod h1:COazNj35u3RXAgQNBr6neR599tYP3URiOpsu9p0rOtk=
|
||||
github.com/go-session/session v3.1.2+incompatible/go.mod h1:8B3iivBQjrz/JtC68Np2T1yBBLxTan3mn/3OM0CyRt0=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
@@ -52,24 +37,30 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
|
||||
github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8=
|
||||
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
@@ -80,7 +71,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
@@ -89,7 +82,9 @@ github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOA
|
||||
github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022 h1:Ys0rDzh8s4UMlGaDa1UTA0sfKgvF0hQZzTYX8ktjiDc=
|
||||
github.com/nullrocks/identicon v0.0.0-20180626043057-7875f45b0022/go.mod h1:x4NsS+uc7ecH/Cbm9xKQ6XzmJM57rWTkjywjfB2yQ18=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -104,7 +99,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
@@ -118,6 +112,8 @@ github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7
|
||||
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
|
||||
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
|
||||
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
|
||||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e h1:+NL1GDIUOKxVfbp2KoJQD9cTQ6dyP2co9q4yzmT9FZo=
|
||||
github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
|
||||
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
|
||||
@@ -127,6 +123,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
|
||||
github.com/valyala/fasthttp v1.0.0 h1:BwIoZQbBsTo3v2F5lz5Oy3TlTq4wLKTLV260EVTEWco=
|
||||
github.com/valyala/fasthttp v1.0.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
|
||||
@@ -139,14 +139,17 @@ github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCO
|
||||
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
|
||||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
|
||||
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
|
||||
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.mongodb.org/mongo-driver v1.1.1 h1:Sq1fR+0c58RME5EoqKdjkiQAmPjmfHlZOoRI6fTUOcs=
|
||||
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff h1:+2zgJKVDVAz/BWSsuniCmU1kLCjL88Z8/kv39xCI9NQ=
|
||||
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -160,7 +163,6 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
@@ -170,7 +172,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY=
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -179,10 +180,10 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -191,6 +192,7 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98M
|
||||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -218,10 +220,13 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/oauth2.v3 v3.10.1 h1:/abis3O6tZFizY1/FgKGoDOmmu2ddvscBSSPdHVa6OI=
|
||||
gopkg.in/oauth2.v3 v3.10.1/go.mod h1:nTG+m2PRcHR9jzGNrGdxSsUKz7vvwkqSlhFrstgZcRU=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
+1
@@ -1,3 +1,4 @@
|
||||
// Package provider implements all oauth2, oauth1 as well as custom and direct providers
|
||||
package provider
|
||||
|
||||
import (
|
||||
|
||||
+1
@@ -1,3 +1,4 @@
|
||||
// Package sender provides email sender
|
||||
package sender
|
||||
|
||||
import (
|
||||
|
||||
+1
@@ -1,3 +1,4 @@
|
||||
// Package token wraps jwt-go library and provides higher level abstraction to work with JWT.
|
||||
package token
|
||||
|
||||
import (
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: true
|
||||
golint:
|
||||
min-confidence: 0
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
maligned:
|
||||
suggest-new: true
|
||||
dupl:
|
||||
threshold: 100
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
misspell:
|
||||
locale: US
|
||||
lll:
|
||||
line-length: 140
|
||||
gocritic:
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- megacheck
|
||||
- govet
|
||||
- unconvert
|
||||
- megacheck
|
||||
- structcheck
|
||||
- gas
|
||||
- gocyclo
|
||||
- dupl
|
||||
- misspell
|
||||
- unparam
|
||||
- varcheck
|
||||
- deadcode
|
||||
- typecheck
|
||||
- ineffassign
|
||||
- varcheck
|
||||
fast: false
|
||||
|
||||
|
||||
run:
|
||||
# modules-download-mode: vendor
|
||||
skip-dirs:
|
||||
- vendor
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- text: "weak cryptographic primitive"
|
||||
linters:
|
||||
- gosec
|
||||
|
||||
service:
|
||||
golangci-lint-version: 1.19.x
|
||||
+1
-1
@@ -9,7 +9,7 @@ go_import_path: github.com/go-pkgz/lcw
|
||||
|
||||
before_install:
|
||||
- export TZ=America/Chicago
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
|
||||
- go get github.com/mattn/goveralls
|
||||
- export PATH=$(pwd)/bin:$PATH
|
||||
|
||||
|
||||
+19
-10
@@ -3,26 +3,26 @@
|
||||
|
||||
The library adds a thin layer on top of [lru cache](https://github.com/hashicorp/golang-lru) and [patrickmn/go-cache](https://github.com/patrickmn/go-cache).
|
||||
|
||||
| Cache name | Constructor | Defaults | Description |
|
||||
| -------------- | --------------------- | ----------------- | --------------------- |
|
||||
| LruCache | lcw.NewLruCache | keys=1000 | LRU cache with limits |
|
||||
| ExpirableCache | lcw.NewExpirableCache | keys=1000, ttl=5m | TTL cache with limits |
|
||||
| Nop | lcw.NewNopCache | | Do-nothing cache |
|
||||
| Cache name | Constructor | Defaults | Description |
|
||||
| -------------- | --------------------- | ----------------- | ----------------------- |
|
||||
| LruCache | lcw.NewLruCache | keys=1000 | LRU cache with limits |
|
||||
| ExpirableCache | lcw.NewExpirableCache | keys=1000, ttl=5m | TTL cache with limits |
|
||||
| RedisCache | lcw.NewRedisCache | ttl=5m | Redis cache with limits |
|
||||
| Nop | lcw.NewNopCache | | Do-nothing cache |
|
||||
|
||||
|
||||
Main features:
|
||||
|
||||
- LoadingCache (guava style)
|
||||
- Limit maximum cache size (in bytes)
|
||||
- Limit maximum key size
|
||||
- Limit maximum size of a value
|
||||
- Limit maximum size of a value
|
||||
- Limit number of keys
|
||||
- TTL support (`ExpirableCache` only)
|
||||
- Callback on eviction event
|
||||
- TTL support (`ExpirableCache` and `RedisCache`)
|
||||
- Callback on eviction event (not supported in `RedisCache`)
|
||||
- Functional style invalidation
|
||||
- Functional options
|
||||
- Sane defaults
|
||||
|
||||
|
||||
## Install and update
|
||||
|
||||
`go get -u github.com/go-pkgz/lcw`
|
||||
@@ -45,6 +45,15 @@ s := val.(string) // cached value
|
||||
|
||||
```
|
||||
|
||||
### Cache with URI
|
||||
|
||||
Cache can be created with URIs:
|
||||
|
||||
- `mem://lru?max_key_size=10&max_val_size=1024&max_keys=50&max_cache_size=64000` - creates LRU cache with given limits
|
||||
- `mem://expirable?ttl=30s&max_key_size=10&max_val_size=1024&max_keys=50&max_cache_size=64000` - create expirable cache
|
||||
- `redis://10.0.0.1:1234?db=16&password=qwerty&network=tcp4&dial_timeout=1s&read_timeout=5s&write_timeout=3s` - create redis cache
|
||||
- `nop://` - create Nop cache
|
||||
|
||||
## Details
|
||||
|
||||
- All byte-size limits (MaxCacheSize and MaxValSize) only work for values implementing `lcw.Sizer` interface.
|
||||
|
||||
+6
-4
@@ -1,12 +1,14 @@
|
||||
// Package lcw adds a thin layer on top of lru cache and go-cache providing more limits and common interface.
|
||||
// The primary method to get (and set) data to/from the cache is LoadingCache.Get retruning stored data for a given key or
|
||||
// call provided func to retrive and store, similar to Guava loading cache.
|
||||
// The primary method to get (and set) data to/from the cache is LoadingCache.Get returning stored data for a given key or
|
||||
// call provided func to retrieve and store, similar to Guava loading cache.
|
||||
// Limits allow max values for key size, number of keys, value size and total size of values in the cache.
|
||||
// CacheStat gives general stats on cache performance.
|
||||
// 3 flavours of cache provided - NoP (do-nothing cache), ExpirableCache (TTL based), and LruCache
|
||||
// 3 flavors of cache provided - NoP (do-nothing cache), ExpirableCache (TTL based), and LruCache
|
||||
package lcw
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Value type wraps interface{}
|
||||
type Value interface{}
|
||||
|
||||
+15
-1
@@ -1,9 +1,23 @@
|
||||
module github.com/go-pkgz/lcw
|
||||
|
||||
require (
|
||||
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
|
||||
github.com/alicebob/miniredis v2.5.0+incompatible
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.0
|
||||
github.com/go-redis/redis/v7 v7.0.0-beta.4
|
||||
github.com/golang/protobuf v1.3.2 // indirect
|
||||
github.com/gomodule/redigo v2.0.0+incompatible // indirect
|
||||
github.com/hashicorp/go-multierror v1.0.0
|
||||
github.com/hashicorp/golang-lru v0.5.3
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/stretchr/testify v1.3.0
|
||||
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
||||
+66
-2
@@ -1,9 +1,42 @@
|
||||
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U=
|
||||
github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
|
||||
github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-redis/redis/v7 v7.0.0-beta.4 h1:p6z7Pde69EGRWvlC++y8aFcaWegyrKHzOBGo0zUACTQ=
|
||||
github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
@@ -13,3 +46,34 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
|
||||
github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0=
|
||||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// LruCache wraps lru.LruCache with laoding cache Get and size limits
|
||||
// LruCache wraps lru.LruCache with loading cache Get and size limits
|
||||
type LruCache struct {
|
||||
options
|
||||
CacheStat
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package lcw
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
redis "github.com/go-redis/redis/v7"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// RedisValueSizeLimit is maximum allowed value size in Redis
|
||||
const RedisValueSizeLimit = 512 * 1024 * 1024
|
||||
|
||||
// RedisCache implements LoadingCache for Redis.
|
||||
type RedisCache struct {
|
||||
options
|
||||
CacheStat
|
||||
backend *redis.Client
|
||||
}
|
||||
|
||||
// NewRedisCache makes Redis LoadingCache implementation.
|
||||
func NewRedisCache(backend *redis.Client, opts ...Option) (*RedisCache, error) {
|
||||
|
||||
res := RedisCache{
|
||||
options: options{
|
||||
ttl: 5 * time.Minute,
|
||||
},
|
||||
}
|
||||
for _, opt := range opts {
|
||||
if err := opt(&res.options); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to set cache option")
|
||||
}
|
||||
}
|
||||
|
||||
if res.maxValueSize <= 0 || res.maxValueSize > RedisValueSizeLimit {
|
||||
res.maxValueSize = RedisValueSizeLimit
|
||||
}
|
||||
|
||||
res.backend = backend
|
||||
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// Get gets value by key or load with fn if not found in cache
|
||||
func (c *RedisCache) Get(key string, fn func() (Value, error)) (data Value, err error) {
|
||||
|
||||
v, getErr := c.backend.Get(key).Result()
|
||||
switch getErr {
|
||||
// RedisClient returns nil when find a key in DB
|
||||
case nil:
|
||||
atomic.AddInt64(&c.Hits, 1)
|
||||
return v, nil
|
||||
// RedisClient returns redis.Nil when doesn't find a key in DB
|
||||
case redis.Nil:
|
||||
if data, err = fn(); err != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, err
|
||||
}
|
||||
// RedisClient returns !nil when something goes wrong while get data
|
||||
default:
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return v, getErr
|
||||
}
|
||||
atomic.AddInt64(&c.Misses, 1)
|
||||
|
||||
if c.allowed(key, data) {
|
||||
_, setErr := c.backend.Set(key, data, c.ttl).Result()
|
||||
if setErr != nil {
|
||||
atomic.AddInt64(&c.Errors, 1)
|
||||
return data, setErr
|
||||
}
|
||||
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Invalidate removes keys with passed predicate fn, i.e. fn(key) should be true to get evicted
|
||||
func (c *RedisCache) Invalidate(fn func(key string) bool) {
|
||||
for _, key := range c.backend.Keys("*").Val() { // Keys() returns copy of cache's key, safe to remove directly
|
||||
if fn(key) {
|
||||
c.backend.Del(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Peek returns the key value (or undefined if not found) without updating the "recently used"-ness of the key.
|
||||
func (c *RedisCache) Peek(key string) (Value, bool) {
|
||||
ret, err := c.backend.Get(key).Result()
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return ret, true
|
||||
}
|
||||
|
||||
// Purge clears the cache completely.
|
||||
func (c *RedisCache) Purge() {
|
||||
c.backend.FlushDB()
|
||||
|
||||
}
|
||||
|
||||
// Delete cache item by key
|
||||
func (c *RedisCache) Delete(key string) {
|
||||
c.backend.Del(key)
|
||||
}
|
||||
|
||||
// Stat returns cache statistics
|
||||
func (c *RedisCache) Stat() CacheStat {
|
||||
return CacheStat{
|
||||
Hits: c.Hits,
|
||||
Misses: c.Misses,
|
||||
Size: c.size(),
|
||||
Keys: c.keys(),
|
||||
Errors: c.Errors,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *RedisCache) size() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *RedisCache) keys() int {
|
||||
return int(c.backend.DBSize().Val())
|
||||
}
|
||||
|
||||
func (c *RedisCache) allowed(key string, data Value) bool {
|
||||
if c.maxKeys > 0 && c.backend.DBSize().Val() >= int64(c.maxKeys) {
|
||||
return false
|
||||
}
|
||||
if c.maxKeySize > 0 && len(key) > c.maxKeySize {
|
||||
return false
|
||||
}
|
||||
if s, ok := data.(Sizer); ok {
|
||||
if c.maxValueSize > 0 && (s.Size() >= c.maxValueSize) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
package lcw
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis/v7"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// New parses uri and makes any of supported caches
|
||||
// supported URIs:
|
||||
// - redis://<ip>:<port>?db=123&max_keys=10
|
||||
// - mem://lru?max_keys=10&max_cache_size=1024
|
||||
// - mem://expirable?ttl=30s&max_val_size=100
|
||||
// - nop://
|
||||
func New(uri string) (LoadingCache, error) {
|
||||
u, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "parse cache uri %s", uri)
|
||||
}
|
||||
|
||||
query := u.Query()
|
||||
opts, err := optionsFromQuery(query)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "parse uri options %s", uri)
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "redis":
|
||||
redisOpts, err := redisOptionsFromURL(u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := NewRedisCache(redis.NewClient(redisOpts), opts...)
|
||||
return res, errors.Wrapf(err, "make redis for %s", uri)
|
||||
case "mem":
|
||||
switch u.Hostname() {
|
||||
case "lru":
|
||||
return NewLruCache(opts...)
|
||||
case "expirable":
|
||||
return NewExpirableCache(opts...)
|
||||
default:
|
||||
return nil, errors.Errorf("unsupported mem cache type %s", u.Hostname())
|
||||
}
|
||||
case "nop":
|
||||
return NewNopCache(), nil
|
||||
}
|
||||
return nil, errors.Errorf("unsupported cache type %s", u.Scheme)
|
||||
}
|
||||
|
||||
func optionsFromQuery(q url.Values) (opts []Option, err error) {
|
||||
|
||||
errs := new(multierror.Error)
|
||||
|
||||
if v := q.Get("max_val_size"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, errors.Wrapf(e, "max_val_size query param %s", v))
|
||||
} else {
|
||||
opts = append(opts, MaxValSize(vv))
|
||||
}
|
||||
}
|
||||
|
||||
if v := q.Get("max_key_size"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, errors.Wrapf(e, "max_key_size query param %s", v))
|
||||
} else {
|
||||
opts = append(opts, MaxKeySize(vv))
|
||||
}
|
||||
}
|
||||
|
||||
if v := q.Get("max_keys"); v != "" {
|
||||
vv, e := strconv.Atoi(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, errors.Wrapf(e, "max_keys query param %s", v))
|
||||
} else {
|
||||
opts = append(opts, MaxKeys(vv))
|
||||
}
|
||||
}
|
||||
|
||||
if v := q.Get("max_cache_size"); v != "" {
|
||||
vv, e := strconv.ParseInt(v, 10, 64)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, errors.Wrapf(e, "max_cache_size query param %s", v))
|
||||
} else {
|
||||
opts = append(opts, MaxCacheSize(vv))
|
||||
}
|
||||
}
|
||||
|
||||
if v := q.Get("ttl"); v != "" {
|
||||
vv, e := time.ParseDuration(v)
|
||||
if e != nil {
|
||||
errs = multierror.Append(errs, errors.Wrapf(e, "ttl query param %s", v))
|
||||
} else {
|
||||
opts = append(opts, TTL(vv))
|
||||
}
|
||||
}
|
||||
|
||||
return opts, errs.ErrorOrNil()
|
||||
}
|
||||
|
||||
func redisOptionsFromURL(u *url.URL) (*redis.Options, error) {
|
||||
query := u.Query()
|
||||
|
||||
db, err := strconv.Atoi(query.Get("db"))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "db from %s", u)
|
||||
}
|
||||
|
||||
res := &redis.Options{
|
||||
Addr: u.Hostname() + ":" + u.Port(),
|
||||
DB: db,
|
||||
Password: query.Get("password"),
|
||||
Network: query.Get("network"),
|
||||
}
|
||||
|
||||
if dialTimeout, err := time.ParseDuration(query.Get("dial_timeout")); err == nil {
|
||||
res.DialTimeout = dialTimeout
|
||||
}
|
||||
|
||||
if readTimeout, err := time.ParseDuration(query.Get("read_timeout")); err == nil {
|
||||
res.ReadTimeout = readTimeout
|
||||
}
|
||||
|
||||
if writeTimeout, err := time.ParseDuration(query.Get("write_timeout")); err == nil {
|
||||
res.WriteTimeout = writeTimeout
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, build with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- "1.11.x"
|
||||
|
||||
install: true
|
||||
|
||||
go_import_path: github.com/go-pkgz/mongo
|
||||
|
||||
services: mongodb
|
||||
|
||||
before_install:
|
||||
- export TZ=America/Chicago
|
||||
- go get gopkg.in/alecthomas/gometalinter.v2
|
||||
- $GOPATH/bin/gometalinter.v2 --install
|
||||
- go get github.com/mattn/goveralls
|
||||
- export PATH=$(pwd)/bin:$PATH
|
||||
- export MONGO_TEST=mongodb://127.0.0.1:27017
|
||||
|
||||
|
||||
script:
|
||||
- GO111MODULE=on go get ./...
|
||||
- GO111MODULE=on go mod vendor
|
||||
- GO111MODULE=on go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
|
||||
- $GOPATH/bin/gometalinter.v2 --deadline=120s --exclude=test --exclude=mock --exclude=vendor --exclude=_example --disable-all --enable=errcheck --enable=vet --enable=vetshadow --enable=megacheck --enable=ineffassign --enable=varcheck --enable=unconvert --enable=deadcode --enable=interfacer --enable=gotype ./... || travis_terminate 1;
|
||||
- $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
# Mongo [](https://travis-ci.org/go-pkgz/mongo) [](https://goreportcard.com/report/github.com/go-pkgz/mongo) [](https://coveralls.io/github/go-pkgz/mongo?branch=master)
|
||||
|
||||
Provides helpers on top of [mgo](https://github.com/globalsign/mgo)
|
||||
|
||||
## Install and update
|
||||
|
||||
`go get -u github.com/go-pkgz/mongo`
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
- `Server` represents mongo instance and provides session accessor. Application usually creates one server object and uses it for anything needed with this particular mongo host or replica set.
|
||||
|
||||
- `Connection` encapsulates session and provides auto-closable wrapper. Each requests runs inside one of With* function makes new mongo session and closes on completion.
|
||||
|
||||
- `BufferedWriter` implements buffered writer to mongo. Write method caching internally till it reached buffer size. Flush methods can be called manually at any time.
|
||||
|
||||
|
||||
```golang
|
||||
m, err := NewServerWithURL("mongodb://127.0.0.1:27017/test?debug=true", 3*time.Second)
|
||||
if err != nil {
|
||||
panic("can't make mongo server")
|
||||
}
|
||||
|
||||
type testRecord struct {
|
||||
Key1 string
|
||||
Kay2 int
|
||||
}
|
||||
|
||||
err = c.WithCollection(func(coll *mgo.Collection) error { // create session
|
||||
// insert 100 records
|
||||
for i := 0; i < 100; i++ {
|
||||
r := testRecord{
|
||||
Key1: fmt.Sprintf("key-%02d", i%5),
|
||||
Key2: i,
|
||||
}
|
||||
if e := coll.Insert(r); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
- [globalsign/mgo](https://github.com/globalsign/mgo) - mgo mongo driver
|
||||
- [stretchr/testify/](https://github.com/stretchr/testify) - testing library (test-only dependency)
|
||||
|
||||
## Testing
|
||||
|
||||
`testing.go` helps to create test for real mongo (not mocks)
|
||||
|
||||
- `mongo.MakeTestConnection` creates `mongo.Connection` for url defined in env `MONGO_TEST`. If not defined `mongodb://mongo:27017` used. By default it will use random connection with prefix `test_` in `test` DB.
|
||||
- `mongo.RemoveTestCollection` - drops collection used by `MakeTestConnection`
|
||||
- `mongo.RemoveTestCollections` - drops user-defined collections from `test` DB
|
||||
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/globalsign/mgo"
|
||||
)
|
||||
|
||||
// sessionFn is a function for all With*Collection calls
|
||||
type sessionFn func(coll *mgo.Collection) error
|
||||
|
||||
// Connection allows to run request in separate session, closing automatically
|
||||
type Connection struct {
|
||||
server *Server
|
||||
db, collection string
|
||||
}
|
||||
|
||||
// NewConnection makes a connection for server
|
||||
func NewConnection(server *Server, db string, collection string) *Connection {
|
||||
return &Connection{server: server, db: db, collection: collection}
|
||||
}
|
||||
|
||||
// WithCollection passes fun with mgo.Collection from session copy, closes it after done,
|
||||
// uses Connection.DB and Connection.Collection
|
||||
func (c *Connection) WithCollection(fun sessionFn) (err error) {
|
||||
return c.WithCustomCollection(c.collection, fun)
|
||||
}
|
||||
|
||||
// WithCustomCollection passes fun with mgo.Collection from session copy, closes it after done
|
||||
// uses Connection.DB or (if not defined) dial.Database, and user-defined collection
|
||||
func (c *Connection) WithCustomCollection(collection string, fun sessionFn) (err error) {
|
||||
db := c.server.dial.Database
|
||||
if c.db != "" {
|
||||
db = c.db
|
||||
}
|
||||
return c.WithCustomDbCollection(db, collection, fun)
|
||||
}
|
||||
|
||||
// WithCustomDbCollection passed fun with mgo.Collection from session copy, closes it after done
|
||||
// uses passed db and collection directly.
|
||||
func (c *Connection) WithCustomDbCollection(db string, collection string, fun sessionFn) (err error) {
|
||||
session := c.server.SessionCopy()
|
||||
defer session.Close()
|
||||
return fun(session.DB(db).C(collection))
|
||||
}
|
||||
|
||||
// WithDB passes fun with mgo.Database from session copy, closes it after done
|
||||
// uses Connection.DB or (if not defined) dial.Database
|
||||
func (c *Connection) WithDB(fun func(dbase *mgo.Database) error) (err error) {
|
||||
db := c.server.dial.Database
|
||||
if c.db != "" {
|
||||
db = c.db
|
||||
}
|
||||
return c.WithCustomDB(db, fun)
|
||||
}
|
||||
|
||||
// WithCustomDB passes fun with mgo.Database from session copy, closes it after done
|
||||
// uses passed db directly
|
||||
func (c *Connection) WithCustomDB(db string, fun func(dbase *mgo.Database) error) (err error) {
|
||||
session := c.server.SessionCopy()
|
||||
defer session.Close()
|
||||
return fun(session.DB(db))
|
||||
}
|
||||
|
||||
func (c *Connection) String() string {
|
||||
return fmt.Sprintf("mongo:%s, db:%s, collection:%s", c.server, c.db, c.collection)
|
||||
}
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
module github.com/go-pkgz/mongo
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731
|
||||
github.com/go-pkgz/lgr v0.2.2
|
||||
github.com/stretchr/objx v0.1.1 // indirect
|
||||
github.com/stretchr/testify v1.3.0
|
||||
)
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731 h1:y7wyeiA6T+TT+HGC9DYypvLkUeg99N4rqHMzn2MmjYk=
|
||||
github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/go-pkgz/lgr v0.1.5 h1:oWj3VNlyYL2uUpdL6Gbi21BAH619Xb/8E78ozpyI2xo=
|
||||
github.com/go-pkgz/lgr v0.1.5/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
|
||||
github.com/go-pkgz/lgr v0.2.2 h1:HSOqMVoetAfvA40Gpy/X/HGyV0UUafIMPgp+SdZends=
|
||||
github.com/go-pkgz/lgr v0.2.2/go.mod h1:hBM1NM/SoYdlrykgdgJWGrZ/TM/XaZIjRbJfx7NkMm8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
-123
@@ -1,123 +0,0 @@
|
||||
// Package mongo wraps mgo to provide easier way to construct mongo server (with auth).
|
||||
// Connection provides With* func wrappers to run query with session copy
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
)
|
||||
|
||||
// Server represents mongo instance and provides session accessor
|
||||
type Server struct {
|
||||
dial mgo.DialInfo
|
||||
params ServerParams
|
||||
sess *mgo.Session
|
||||
}
|
||||
|
||||
// ServerParams optional set of parameters
|
||||
type ServerParams struct {
|
||||
ConsistencyMode mgo.Mode
|
||||
Delay int // initial delay to give mongo server some time to start, in case if mongo part of the same compose
|
||||
Debug bool // turn on mgo debug mode
|
||||
SSL bool // enforce SSL connection
|
||||
}
|
||||
|
||||
// NewServerWithURL makes mongo server from url like
|
||||
// mongodb://remark42:password@127.0.0.1:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin
|
||||
func NewServerWithURL(url string, timeout time.Duration) (res *Server, err error) {
|
||||
dial, params, err := parseURL(url, timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create mongo server from url %s, %s", url, err)
|
||||
}
|
||||
return NewServer(dial, params)
|
||||
}
|
||||
|
||||
// NewServer doing auth if passwd != "" and can delay to make sure local mongo is up
|
||||
func NewServer(dial mgo.DialInfo, params ServerParams) (res *Server, err error) {
|
||||
result := Server{dial: dial, params: params}
|
||||
|
||||
if params.Debug {
|
||||
mgo.SetDebug(true)
|
||||
mgo.SetLogger(&mgdLogger{})
|
||||
}
|
||||
|
||||
if len(dial.Addrs) == 0 {
|
||||
return nil, errors.New("missing mongo address")
|
||||
}
|
||||
|
||||
if params.Delay > 0 {
|
||||
log.Printf("[DEBUG] initial mongo delay=%d", params.Delay)
|
||||
time.Sleep(time.Duration(params.Delay) * time.Second)
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] dial mongo %s, ssl=%v", dial.Addrs, params.SSL)
|
||||
|
||||
if params.SSL {
|
||||
tlsConfig := &tls.Config{}
|
||||
dial.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
|
||||
conn, e := tls.Dial("tcp", addr.String(), tlsConfig)
|
||||
return conn, e
|
||||
}
|
||||
}
|
||||
|
||||
session, err := mgo.DialWithInfo(&dial)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't connect to mongo, %v", err)
|
||||
}
|
||||
session.SetMode(params.ConsistencyMode, true)
|
||||
session.SetSyncTimeout(30 * time.Second)
|
||||
session.SetSocketTimeout(dial.Timeout)
|
||||
|
||||
if dial.Username != "" && dial.Password != "" {
|
||||
creds := &mgo.Credential{Username: dial.Username, Password: dial.Password, Source: dial.Source}
|
||||
log.Printf("[DEBUG] login to mongo, user=%s, db=%s", creds.Username, creds.Source)
|
||||
if err = session.Login(creds); err != nil {
|
||||
return nil, fmt.Errorf("can't login to mongo, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
result.sess = session
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// SessionCopy returns copy of main session. Client should close it
|
||||
func (m *Server) SessionCopy() *mgo.Session {
|
||||
return m.sess.Copy()
|
||||
}
|
||||
|
||||
func (m *Server) String() string {
|
||||
return fmt.Sprintf("%v%s", m.dial.Addrs, m.dial.Database)
|
||||
}
|
||||
|
||||
// parseURL extends mgo with debug option and extracts ssl flag to make ServerParams
|
||||
func parseURL(mongoURL string, connectTimeout time.Duration) (mgo.DialInfo, ServerParams, error) {
|
||||
params := ServerParams{
|
||||
ConsistencyMode: mgo.Monotonic,
|
||||
SSL: strings.Contains(mongoURL, "ssl=true"),
|
||||
Debug: strings.Contains(mongoURL, "debug=true"),
|
||||
}
|
||||
|
||||
mongoURL = strings.Replace(mongoURL, "&debug=true", "", 1)
|
||||
mongoURL = strings.Replace(mongoURL, "?debug=true", "", 1)
|
||||
|
||||
dial, err := mgo.ParseURL(mongoURL)
|
||||
if err != nil {
|
||||
return mgo.DialInfo{}, ServerParams{}, fmt.Errorf("failed to parse mongo url %s, %s", mongoURL, err)
|
||||
}
|
||||
dial.Timeout = connectTimeout
|
||||
return *dial, params, nil
|
||||
}
|
||||
|
||||
type mgdLogger struct{}
|
||||
|
||||
func (l *mgdLogger) Output(calldepth int, s string) error {
|
||||
log.Printf("[DEBUG] MGO %s", s)
|
||||
return nil
|
||||
}
|
||||
-76
@@ -1,76 +0,0 @@
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var conn *Connection
|
||||
var once sync.Once
|
||||
|
||||
// MakeTestConnection connects to MONGO_TEST url or "mongo" host (in no env) and returns new connection.
|
||||
// collection name randomized on each call
|
||||
func MakeTestConnection(t *testing.T) (*Connection, error) {
|
||||
mongoURL := getMongoURL(t)
|
||||
once.Do(func() {
|
||||
log.Print("[DEBUG] connect to mongo test instance")
|
||||
srv, err := NewServerWithURL(mongoURL, 10*time.Second)
|
||||
assert.Nil(t, err, "failed to dial")
|
||||
collName := fmt.Sprintf("test_%d", time.Now().Nanosecond())
|
||||
conn = NewConnection(srv, "test", collName)
|
||||
})
|
||||
RemoveTestCollection(t, conn)
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// RemoveTestCollection removes all records and drop collection from connection
|
||||
func RemoveTestCollection(t *testing.T, c *Connection) {
|
||||
log.Printf("[DEBUG] clean test collection %+v", c.collection)
|
||||
_ = c.WithCollection(func(coll *mgo.Collection) error {
|
||||
_, e := coll.RemoveAll(nil)
|
||||
require.Nil(t, e, "failed to remove records, %s", e)
|
||||
e = coll.DropCollection()
|
||||
if e != nil && e.Error() != "ns not found" {
|
||||
require.Nil(t, e, "failed to drop collection, %s", e)
|
||||
}
|
||||
return e
|
||||
})
|
||||
}
|
||||
|
||||
// RemoveTestCollections clears passed collections
|
||||
func RemoveTestCollections(t *testing.T, c *Connection, collections ...string) {
|
||||
log.Printf("[DEBUG] clean test collections %+v", collections)
|
||||
for _, collection := range collections {
|
||||
_ = c.WithCustomCollection(collection, func(coll *mgo.Collection) error {
|
||||
_, e := coll.RemoveAll(nil)
|
||||
require.Nil(t, e, "failed to remove records, %s", e)
|
||||
e = coll.DropCollection()
|
||||
if e != nil && e.Error() != "ns not found" {
|
||||
require.Nil(t, e, "failed to drop collection, %s", e)
|
||||
}
|
||||
return e
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getMongoURL(t *testing.T) string {
|
||||
mongoURL := os.Getenv("MONGO_TEST")
|
||||
if mongoURL == "" {
|
||||
mongoURL = "mongodb://mongo:27017"
|
||||
t.Logf("no MONGO_TEST in env, defaulted to %s", mongoURL)
|
||||
}
|
||||
if mongoURL == "skip" {
|
||||
t.Skip("skip mongo test")
|
||||
}
|
||||
return mongoURL
|
||||
}
|
||||
-154
@@ -1,154 +0,0 @@
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/globalsign/mgo"
|
||||
log "github.com/go-pkgz/lgr"
|
||||
)
|
||||
|
||||
// BufferedWriter defines interface for writes and flush
|
||||
type BufferedWriter interface {
|
||||
Write(rec interface{}) error
|
||||
Flush() error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// BufferedWriterMgo collects records in local buffer and flushes them as filled. Thread safe
|
||||
// by default using both DB and collection from provided connection.
|
||||
// Collection can be customized by WithCollection method. Optional flush duration to save on interval
|
||||
type BufferedWriterMgo struct {
|
||||
connection *Connection
|
||||
bufferSize int
|
||||
collection string
|
||||
flushDuration time.Duration
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
buffer []interface{}
|
||||
lock sync.Mutex
|
||||
lastWriteTime time.Time
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// NewBufferedWriter makes batch writer for given size and connection
|
||||
func NewBufferedWriter(size int, connection *Connection) *BufferedWriterMgo {
|
||||
if size == 0 {
|
||||
size = 1
|
||||
}
|
||||
return &BufferedWriterMgo{
|
||||
bufferSize: size,
|
||||
buffer: make([]interface{}, 0, size+1),
|
||||
connection: connection,
|
||||
}
|
||||
}
|
||||
|
||||
// WithCollection sets custom collection to use with writer
|
||||
func (bw *BufferedWriterMgo) WithCollection(collection string) *BufferedWriterMgo {
|
||||
bw.collection = collection
|
||||
return bw
|
||||
}
|
||||
|
||||
// WithAutoFlush sets auto flush duration
|
||||
func (bw *BufferedWriterMgo) WithAutoFlush(duration time.Duration) *BufferedWriterMgo {
|
||||
bw.flushDuration = duration
|
||||
if duration > 0 { // activate background auto-flush
|
||||
bw.once.Do(func() {
|
||||
bw.ctx, bw.cancel = context.WithCancel(context.Background())
|
||||
ticker := time.NewTicker(duration)
|
||||
go func() {
|
||||
defer bw.cancel()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
var shouldFlush bool
|
||||
_ = bw.synced(func() error {
|
||||
shouldFlush = time.Now().After(bw.lastWriteTime.Add(bw.flushDuration)) && len(bw.buffer) > 0
|
||||
return nil
|
||||
})
|
||||
if shouldFlush {
|
||||
if err := bw.Flush(); err != nil {
|
||||
log.Printf("[WARN] flush failed, %s", err)
|
||||
}
|
||||
}
|
||||
case <-bw.ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
return bw
|
||||
}
|
||||
|
||||
// Write to buffer and, as filled, to mongo. If flushDuration defined check for automatic flush
|
||||
func (bw *BufferedWriterMgo) Write(rec interface{}) error {
|
||||
return bw.synced(func() error {
|
||||
bw.lastWriteTime = time.Now()
|
||||
bw.buffer = append(bw.buffer, rec)
|
||||
if len(bw.buffer) >= bw.bufferSize {
|
||||
if err := bw.writeBuffer(); err != nil {
|
||||
return fmt.Errorf("failed to write to %s, %s", bw.connection, err)
|
||||
}
|
||||
bw.buffer = bw.buffer[0:0]
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// Flush writes everything left in buffer to mongo
|
||||
func (bw *BufferedWriterMgo) Flush() error {
|
||||
err := bw.synced(func() error {
|
||||
err := bw.writeBuffer()
|
||||
bw.buffer = bw.buffer[0:0]
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to flush to %s, %s", bw.connection, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close flushes all in-fly records and terminates background auto-flusher
|
||||
func (bw *BufferedWriterMgo) Close() (err error) {
|
||||
return bw.synced(func() error {
|
||||
err = bw.writeBuffer()
|
||||
if bw.flushDuration > 0 {
|
||||
bw.cancel()
|
||||
<-bw.ctx.Done()
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// writeBuffer sends all collected records to mongo
|
||||
func (bw *BufferedWriterMgo) writeBuffer() (err error) {
|
||||
|
||||
if len(bw.buffer) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if bw.collection == "" { // no custom collection
|
||||
err = bw.connection.WithCollection(func(coll *mgo.Collection) error {
|
||||
return coll.Insert(bw.buffer...)
|
||||
})
|
||||
}
|
||||
|
||||
if bw.collection != "" { // with custom collection
|
||||
err = bw.connection.WithCustomCollection(bw.collection, func(coll *mgo.Collection) error {
|
||||
return coll.Insert(bw.buffer...)
|
||||
})
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (bw *BufferedWriterMgo) synced(fn func() error) error {
|
||||
bw.lock.Lock()
|
||||
defer bw.lock.Unlock()
|
||||
return fn()
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: true
|
||||
golint:
|
||||
min-confidence: 0
|
||||
gocyclo:
|
||||
min-complexity: 15
|
||||
maligned:
|
||||
suggest-new: true
|
||||
dupl:
|
||||
threshold: 100
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 2
|
||||
misspell:
|
||||
locale: US
|
||||
lll:
|
||||
line-length: 140
|
||||
gocritic:
|
||||
enabled-tags:
|
||||
- performance
|
||||
- style
|
||||
- experimental
|
||||
disabled-checks:
|
||||
- wrapperFunc
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- megacheck
|
||||
- govet
|
||||
- unconvert
|
||||
- megacheck
|
||||
- structcheck
|
||||
- gas
|
||||
- gocyclo
|
||||
- dupl
|
||||
- misspell
|
||||
- unparam
|
||||
- varcheck
|
||||
- deadcode
|
||||
- typecheck
|
||||
- ineffassign
|
||||
- varcheck
|
||||
fast: false
|
||||
|
||||
|
||||
run:
|
||||
# modules-download-mode: vendor
|
||||
skip-dirs:
|
||||
- vendor
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- text: "weak cryptographic primitive"
|
||||
linters:
|
||||
- gosec
|
||||
|
||||
service:
|
||||
golangci-lint-version: 1.16.x
|
||||
+11
-7
@@ -1,16 +1,20 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- "1.11.x"
|
||||
- "1.12.x"
|
||||
|
||||
go_import_path: github.com/go-pkgz/repeater
|
||||
install: true
|
||||
|
||||
before_install:
|
||||
- export TZ=America/Chicago
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.2
|
||||
- go get github.com/mattn/goveralls
|
||||
- go get gopkg.in/alecthomas/gometalinter.v2
|
||||
- $GOPATH/bin/gometalinter.v2 --install
|
||||
- export PATH=$(pwd)/bin:$PATH
|
||||
|
||||
script:
|
||||
- go test ./...
|
||||
- $GOPATH/bin/gometalinter.v2 --exclude=test --exclude=mock --exclude=vendor ./...
|
||||
- $GOPATH/bin/goveralls -service=travis-ci
|
||||
- GO111MODULE=on go get ./...
|
||||
- GO111MODULE=on go mod vendor
|
||||
- GO111MODULE=on go test -v -mod=vendor -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
|
||||
- GO111MODULE=on go test -v -covermode=count -coverprofile=profile.cov ./... || travis_terminate 1;
|
||||
- golangci-lint run || travis_terminate 1;
|
||||
- $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 Umputun
|
||||
Copyright (c) 2019 Umputun
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
+4
@@ -1 +1,5 @@
|
||||
module github.com/go-pkgz/repeater
|
||||
|
||||
go 1.12
|
||||
|
||||
require github.com/stretchr/testify v1.3.0
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
+1
-2
@@ -45,7 +45,6 @@ func (r Repeater) Do(ctx context.Context, fun func() error, errors ...error) (er
|
||||
}
|
||||
|
||||
ch := r.Start(ctx) // channel of ticks-like events provided by strategy
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -57,7 +56,7 @@ func (r Repeater) Do(ctx context.Context, fun func() error, errors ...error) (er
|
||||
if err = fun(); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err != nil && inErrors(err) { //terminate on critical error from provided list
|
||||
if err != nil && inErrors(err) { // terminate on critical error from provided list
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
+10
-10
@@ -9,7 +9,7 @@ import (
|
||||
)
|
||||
|
||||
// Backoff implements strategy.Interface for exponential-backoff
|
||||
// it starts from 100ms and goes in steps with last * math.Pow(factor, attempt)
|
||||
// it starts from 100ms (by default, if no Duration set) and goes in steps with last * math.Pow(factor, attempt)
|
||||
// optional jitter randomize intervals a little bit.
|
||||
type Backoff struct {
|
||||
Duration time.Duration
|
||||
@@ -23,7 +23,7 @@ type Backoff struct {
|
||||
// Start returns channel, similar to time.Timer
|
||||
// then publishing signals to channel ch for retries attempt. Closed ch indicates "done" event
|
||||
// consumer (repeater) should stop it explicitly after completion
|
||||
func (b *Backoff) Start(ctx context.Context) (ch chan struct{}) {
|
||||
func (b *Backoff) Start(ctx context.Context) <-chan struct{} {
|
||||
|
||||
b.once.Do(func() {
|
||||
if b.Duration == 0 {
|
||||
@@ -37,7 +37,7 @@ func (b *Backoff) Start(ctx context.Context) (ch chan struct{}) {
|
||||
}
|
||||
})
|
||||
|
||||
ch = make(chan struct{})
|
||||
ch := make(chan struct{})
|
||||
go func() {
|
||||
defer close(ch)
|
||||
rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
|
||||
@@ -45,14 +45,14 @@ func (b *Backoff) Start(ctx context.Context) (ch chan struct{}) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
ch <- struct{}{}
|
||||
delay := float64(b.Duration) * math.Pow(b.Factor, float64(i))
|
||||
if b.Jitter {
|
||||
delay = rnd.Float64()*(float64(2*b.Duration)) + (delay - float64(b.Duration))
|
||||
}
|
||||
sleep(ctx, time.Duration(delay))
|
||||
case ch <- struct{}{}:
|
||||
}
|
||||
|
||||
delay := float64(b.Duration) * math.Pow(b.Factor, float64(i))
|
||||
if b.Jitter {
|
||||
delay = rnd.Float64()*(float64(2*b.Duration)) + (delay - float64(b.Duration))
|
||||
}
|
||||
sleep(ctx, time.Duration(delay))
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
|
||||
+7
-6
@@ -14,21 +14,22 @@ type FixedDelay struct {
|
||||
// Start returns channel, similar to time.Timer
|
||||
// then publishing signals to channel ch for retries attempt.
|
||||
// can be terminated (canceled) via context.
|
||||
func (s *FixedDelay) Start(ctx context.Context) (ch chan struct{}) {
|
||||
func (s *FixedDelay) Start(ctx context.Context) <-chan struct{} {
|
||||
if s.Repeats == 0 {
|
||||
s.Repeats = 1
|
||||
}
|
||||
ch = make(chan struct{})
|
||||
ch := make(chan struct{})
|
||||
go func() {
|
||||
defer close(ch)
|
||||
defer func() {
|
||||
close(ch)
|
||||
}()
|
||||
for i := 0; i < s.Repeats; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
ch <- struct{}{}
|
||||
sleep(ctx, s.Delay)
|
||||
case ch <- struct{}{}:
|
||||
}
|
||||
sleep(ctx, s.Delay)
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
|
||||
+3
-3
@@ -9,15 +9,15 @@ import (
|
||||
|
||||
// Interface for repeater strategy. Returns channel with ticks
|
||||
type Interface interface {
|
||||
Start(ctx context.Context) chan struct{}
|
||||
Start(ctx context.Context) <-chan struct{}
|
||||
}
|
||||
|
||||
// Once strategy eliminate repeats and makes a single try only
|
||||
type Once struct{}
|
||||
|
||||
// Start returns closed channel with a single element to prevent any repeats
|
||||
func (s *Once) Start(ctx context.Context) (ch chan struct{}) {
|
||||
ch = make(chan struct{})
|
||||
func (s *Once) Start(ctx context.Context) <-chan struct{} {
|
||||
ch := make(chan struct{})
|
||||
go func() {
|
||||
ch <- struct{}{}
|
||||
close(ch)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
*.rdb
|
||||
testdata/*/
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
run:
|
||||
concurrency: 8
|
||||
deadline: 5m
|
||||
tests: false
|
||||
linters:
|
||||
enable-all: true
|
||||
disable:
|
||||
- gochecknoglobals
|
||||
- goconst
|
||||
- gosec
|
||||
- maligned
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
dist: xenial
|
||||
sudo: false
|
||||
language: go
|
||||
|
||||
services:
|
||||
- redis-server
|
||||
|
||||
go:
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
- tip
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- go: tip
|
||||
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
|
||||
go_import_path: github.com/go-redis/redis
|
||||
|
||||
before_install:
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# Changelog
|
||||
|
||||
## v7 WIP
|
||||
|
||||
- WrapProcess is replaced with more convenient AddHook that has access to context.Context.
|
||||
- WithContext now can not be used to create a shallow copy of the client.
|
||||
- New methods ProcessContext, DoContext, and ExecContext.
|
||||
- Client respects Context.Deadline when setting net.Conn deadline.
|
||||
- Client listens on Context.Done while waiting for a connection from the pool and returns an error when context context is cancelled.
|
||||
- Add PubSub.ChannelWithSubscriptions that sends `*Subscription` in addition to `*Message` to allow detecting reconnections.
|
||||
- `time.Time` is now marshalled in RFC3339 format. `rdb.Get("foo").Time()` helper is added to parse time.
|
||||
|
||||
## v6.15
|
||||
|
||||
- Cluster and Ring pipelines process commands for each node in its own goroutine.
|
||||
|
||||
## 6.14
|
||||
|
||||
- Added Options.MinIdleConns.
|
||||
- Added Options.MaxConnAge.
|
||||
- PoolStats.FreeConns is renamed to PoolStats.IdleConns.
|
||||
- Add Client.Do to simplify creating custom commands.
|
||||
- Add Cmd.String, Cmd.Int, Cmd.Int64, Cmd.Uint64, Cmd.Float64, and Cmd.Bool helpers.
|
||||
- Lower memory usage.
|
||||
|
||||
## v6.13
|
||||
|
||||
- Ring got new options called `HashReplicas` and `Hash`. It is recommended to set `HashReplicas = 1000` for better keys distribution between shards.
|
||||
- Cluster client was optimized to use much less memory when reloading cluster state.
|
||||
- PubSub.ReceiveMessage is re-worked to not use ReceiveTimeout so it does not lose data when timeout occurres. In most cases it is recommended to use PubSub.Channel instead.
|
||||
- Dialer.KeepAlive is set to 5 minutes by default.
|
||||
|
||||
## v6.12
|
||||
|
||||
- ClusterClient got new option called `ClusterSlots` which allows to build cluster of normal Redis Servers that don't have cluster mode enabled. See https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
Copyright (c) 2013 The github.com/go-redis/redis Authors.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
all: testdeps
|
||||
go test ./...
|
||||
go test ./... -short -race
|
||||
go test ./... -run=NONE -bench=. -benchmem
|
||||
env GOOS=linux GOARCH=386 go test ./...
|
||||
go vet
|
||||
go get github.com/gordonklaus/ineffassign
|
||||
ineffassign .
|
||||
golangci-lint run
|
||||
|
||||
testdeps: testdata/redis/src/redis-server
|
||||
|
||||
bench: testdeps
|
||||
go test ./... -test.run=NONE -test.bench=. -test.benchmem
|
||||
|
||||
.PHONY: all test testdeps bench
|
||||
|
||||
testdata/redis:
|
||||
mkdir -p $@
|
||||
wget -qO- https://github.com/antirez/redis/archive/5.0.tar.gz | tar xvz --strip-components=1 -C $@
|
||||
|
||||
testdata/redis/src/redis-server: testdata/redis
|
||||
sed -i.bak 's/libjemalloc.a/libjemalloc.a -lrt/g' $</src/Makefile
|
||||
cd $< && make all
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
# Redis client for Golang
|
||||
|
||||
[](https://travis-ci.org/go-redis/redis)
|
||||
[](https://godoc.org/github.com/go-redis/redis)
|
||||
[](https://airbrake.io)
|
||||
|
||||
Supports:
|
||||
|
||||
- Redis 3 commands except QUIT, MONITOR, SLOWLOG and SYNC.
|
||||
- Automatic connection pooling with [circuit breaker](https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern) support.
|
||||
- [Pub/Sub](https://godoc.org/github.com/go-redis/redis#PubSub).
|
||||
- [Transactions](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline).
|
||||
- [Pipeline](https://godoc.org/github.com/go-redis/redis#example-Client-Pipeline) and [TxPipeline](https://godoc.org/github.com/go-redis/redis#example-Client-TxPipeline).
|
||||
- [Scripting](https://godoc.org/github.com/go-redis/redis#Script).
|
||||
- [Timeouts](https://godoc.org/github.com/go-redis/redis#Options).
|
||||
- [Redis Sentinel](https://godoc.org/github.com/go-redis/redis#NewFailoverClient).
|
||||
- [Redis Cluster](https://godoc.org/github.com/go-redis/redis#NewClusterClient).
|
||||
- [Cluster of Redis Servers](https://godoc.org/github.com/go-redis/redis#example-NewClusterClient--ManualSetup) without using cluster mode and Redis Sentinel.
|
||||
- [Ring](https://godoc.org/github.com/go-redis/redis#NewRing).
|
||||
- [Instrumentation](https://godoc.org/github.com/go-redis/redis#ex-package--Instrumentation).
|
||||
- [Cache friendly](https://github.com/go-redis/cache).
|
||||
- [Rate limiting](https://github.com/go-redis/redis_rate).
|
||||
- [Distributed Locks](https://github.com/bsm/redislock).
|
||||
|
||||
API docs: https://godoc.org/github.com/go-redis/redis.
|
||||
Examples: https://godoc.org/github.com/go-redis/redis#pkg-examples.
|
||||
|
||||
## Installation
|
||||
|
||||
Install:
|
||||
|
||||
```shell
|
||||
go get -u github.com/go-redis/redis
|
||||
```
|
||||
|
||||
Import:
|
||||
|
||||
```go
|
||||
import "github.com/go-redis/redis"
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
```go
|
||||
func ExampleNewClient() {
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: "localhost:6379",
|
||||
Password: "", // no password set
|
||||
DB: 0, // use default DB
|
||||
})
|
||||
|
||||
pong, err := client.Ping().Result()
|
||||
fmt.Println(pong, err)
|
||||
// Output: PONG <nil>
|
||||
}
|
||||
|
||||
func ExampleClient() {
|
||||
err := client.Set("key", "value", 0).Err()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
val, err := client.Get("key").Result()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println("key", val)
|
||||
|
||||
val2, err := client.Get("key2").Result()
|
||||
if err == redis.Nil {
|
||||
fmt.Println("key2 does not exist")
|
||||
} else if err != nil {
|
||||
panic(err)
|
||||
} else {
|
||||
fmt.Println("key2", val2)
|
||||
}
|
||||
// Output: key value
|
||||
// key2 does not exist
|
||||
}
|
||||
```
|
||||
|
||||
## Howto
|
||||
|
||||
Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package.
|
||||
|
||||
## Look and feel
|
||||
|
||||
Some corner cases:
|
||||
|
||||
```go
|
||||
// SET key value EX 10 NX
|
||||
set, err := client.SetNX("key", "value", 10*time.Second).Result()
|
||||
|
||||
// SORT list LIMIT 0 2 ASC
|
||||
vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result()
|
||||
|
||||
// ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2
|
||||
vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{
|
||||
Min: "-inf",
|
||||
Max: "+inf",
|
||||
Offset: 0,
|
||||
Count: 2,
|
||||
}).Result()
|
||||
|
||||
// ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM
|
||||
vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result()
|
||||
|
||||
// EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello"
|
||||
vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result()
|
||||
|
||||
// custom command
|
||||
res, err := client.Do("set", "key", "value")
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Golang PostgreSQL ORM](https://github.com/go-pg/pg)
|
||||
- [Golang msgpack](https://github.com/vmihailenco/msgpack)
|
||||
- [Golang message task queue](https://github.com/vmihailenco/taskq)
|
||||
+1624
File diff suppressed because it is too large
Load Diff
+22
@@ -0,0 +1,22 @@
|
||||
package redis
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
func (c *ClusterClient) DBSize() *IntCmd {
|
||||
cmd := NewIntCmd("dbsize")
|
||||
var size int64
|
||||
err := c.ForEachMaster(func(master *Client) error {
|
||||
n, err := master.DBSize().Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
atomic.AddInt64(&size, n)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
cmd.setErr(err)
|
||||
return cmd
|
||||
}
|
||||
cmd.val = size
|
||||
return cmd
|
||||
}
|
||||
+1919
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user