feature/mongo cache (#180)

* add siteID to cache Get * indirect option setters * add mongo cache with tests, add Key and Flusher * lint: minor warns * workaround for cache parallel test * repeater in mongo cache * missing repeater vendor * fix nop cache * add cache mongo benchmark * wired mongo cache, single opts group for mongo * disable goconst * stop cache repeated on not found error * use local mongo for tests in travis
2018-07-24 18:43:53 -04:00
parent dbd1d4069f
commit 3520de768d
27 changed files with 967 additions and 150 deletions
@@ -0,0 +1,12 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
@@ -0,0 +1,18 @@
+language: go
+
+go:
+  - "1.10.x"
+
+go_import_path: github.com/go-pkgz/repeater
+
+services: mongodb
+
+before_install:
+  - go get github.com/mattn/goveralls
+  - go get gopkg.in/alecthomas/gometalinter.v2
+  - $GOPATH/bin/gometalinter.v2 --install
+
+script:
+  - go test ./...
+  - $GOPATH/bin/gometalinter.v2 --exclude=test --exclude=mock --exclude=vendor ./...
+  - $GOPATH/bin/goveralls -service=travis-ci
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Umputun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
@@ -0,0 +1,36 @@
+# Repeater [![Build Status](https://travis-ci.org/go-pkgz/repeater.svg?branch=master)](https://travis-ci.org/go-pkgz/repeater) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/repeater)](https://goreportcard.com/report/github.com/go-pkgz/repeater) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/repeater/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/repeater?branch=master)
+
+Repeater calls a function until it returns no error, up to some number of iterations and delays defined by strategy. It terminates immediately on err from the provided (optional) list of critical errors.
+
+## Install and update
+
+`go get -u github.com/go-pkgz/repeater`
+
+## How to use
+
+New Repeater created by `New(strtg strategy.Interface)` or shortcut for defaults - `NewDefault(repeats int, delay time.Duration) *Repeater`.
+
+To activate invoke `Do` method. `Do` repeats func until no error returned. Predefined (optional) errors terminates the loop immediately.
+                            
+`func (r Repeater) Do(fun func() error, errors ...error) (err error)`
+
+### Repeating strategy
+
+User can provide his own strategy implementing the interface:
+
+```go
+type Interface interface {
+	Start(ctx context.Context) chan struct{}
+}
+```
+
+Returned channels used as "ticks," i.e., for each repeat or initial operation one read from this channel needed. Closing this channel indicates "done with retries." It is pretty much the same idea as `time.Timer` or `time.Tick` implements. Note - the first (technically not-repeated-yet) call won't happen **until something sent to the channel**. For this reason, the typical strategy sends first "tick" before the first wait/sleep.
+
+Three most common strategies provided by package and ready to use:
+1. **Fixed delay**, up to max number of attempts - `NewFixedDelay(repeats int, delay time.Duration)`. 
+It is the default strategy used by `repeater.NewDefault` constructor
+2. **BackOff** with jitter provides exponential backoff. It starts from 100ms interval and goes in steps with `last * math.Pow(factor, attempt)`. Optional jitter randomizes intervals a little bit. The strategy created by `NewBackoff(repeats int, factor float64, jitter bool)`. _Factor = 1 effectively makes this strategy fixed with 100ms delay._ 
+3. **Once** strategy does not do any repeats and mainly used for tests/mocks - `NewOnce()`
+
+
+
@@ -0,0 +1,60 @@
+// Package repeater call fun till it returns no error, up to repeat some number of iterations and delays defined by strategy.
+// Repeats number and delays defined by strategy.Interface. Terminates immediately on err from
+// provided, optional list of critical errors
+package repeater
+
+import (
+	"context"
+	"time"
+
+	"github.com/go-pkgz/repeater/strategy"
+)
+
+// Repeater is the main object, should be made by New or NewDefault, embeds strategy
+type Repeater struct {
+	strategy.Interface
+}
+
+// New repeater with a given strategy. If strategy=nil initializes with FixedDelay 5sec, 10 times.
+func New(strtg strategy.Interface) *Repeater {
+	if strtg == nil {
+		strtg = strategy.NewFixedDelay(10, time.Second*5)
+	}
+	result := Repeater{Interface: strtg}
+	return &result
+}
+
+// NewDefault makes repeater with FixedDelay strategy
+func NewDefault(repeats int, delay time.Duration) *Repeater {
+	return New(strategy.NewFixedDelay(repeats, delay))
+}
+
+// Do repeats fun till no error. Predefined (optional) errors terminate immediately
+func (r Repeater) Do(fun func() error, errors ...error) (err error) {
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc() // ensure strategy's channel termination
+
+	inErrors := func(err error) bool {
+		for _, e := range errors {
+			if e == err {
+				return true
+			}
+		}
+		return false
+	}
+
+	ch := r.Start(ctx) // channel of ticks-like events provided by strategy
+
+	// closed channel indicates completion or early termination, set by strategy
+	for range ch {
+
+		if err = fun(); err == nil {
+			return nil
+		}
+		if err != nil && inErrors(err) { //terminate on critical error from provided list
+			return err
+		}
+	}
+	return err
+}
@@ -0,0 +1,55 @@
+package strategy
+
+import (
+	"context"
+	"math"
+	"math/rand"
+	"time"
+)
+
+// Backoff implements strategy.Interface for exponential-backoff
+// it starts from 100ms and goes in steps with last * math.Pow(factor, attempt)
+// optional jitter randomize intervals a little bit.
+type Backoff struct {
+	repeats int
+	factor  float64
+	jitter  bool
+}
+
+// NewBackoff makes Backoff strategy with given factor and optional jitter
+func NewBackoff(repeats int, factor float64, jitter bool) Interface {
+	if repeats == 0 {
+		repeats = 1
+	}
+	if factor <= 0 {
+		factor = 1
+	}
+	result := Backoff{repeats: repeats, factor: factor, jitter: jitter}
+	return &result
+}
+
+// Start returns channel, similar to time.Timer
+// then publishing signals to channel ch for retries attempt. Closed ch indicates "done" event
+// consumer (repeater) should stop it explicitly after completion
+func (b *Backoff) Start(ctx context.Context) (ch chan struct{}) {
+	ch = make(chan struct{})
+	go func() {
+		defer close(ch)
+		rnd := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
+		minDelay := 100 * time.Millisecond // starts 100ms
+		for i := 0; i < b.repeats; i++ {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				ch <- struct{}{}
+				delay := float64(minDelay) * math.Pow(b.factor, float64(i))
+				if b.jitter {
+					delay = rnd.Float64()*(float64(2*minDelay)) + (delay - float64(minDelay))
+				}
+				time.Sleep(time.Duration(delay))
+			}
+		}
+	}()
+	return ch
+}
@@ -0,0 +1,41 @@
+package strategy
+
+import (
+	"context"
+	"time"
+)
+
+// FixedDelay implements strategy.Interface for fixed intervals up to max repeats
+type FixedDelay struct {
+	repeats int
+	delay   time.Duration
+}
+
+// NewFixedDelay makes a Interface
+func NewFixedDelay(repeats int, delay time.Duration) Interface {
+	if repeats == 0 {
+		repeats = 1
+	}
+	result := FixedDelay{repeats: repeats, delay: delay}
+	return &result
+}
+
+// Start returns channel, similar to time.Timer
+// then publishing signals to channel ch for retries attempt.
+// can be terminated (canceled) via context.
+func (s *FixedDelay) Start(ctx context.Context) (ch chan struct{}) {
+	ch = make(chan struct{})
+	go func() {
+		defer close(ch)
+		for i := 0; i < s.repeats; i++ {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				ch <- struct{}{}
+				time.Sleep(s.delay)
+			}
+		}
+	}()
+	return ch
+}
@@ -0,0 +1,28 @@
+// Package strategy defines repeater's strategy and implements some.
+// Strategy result is a channel acting like time.Timer ot time.Tick
+package strategy
+
+import "context"
+
+// Interface for repeater strategy. Returns channel with ticks
+type Interface interface {
+	Start(ctx context.Context) chan struct{}
+}
+
+// Once strategy eliminate repeats and makes a single try only
+type Once struct{}
+
+// NewOnce makes no-repeat strategy
+func NewOnce() Interface {
+	return &Once{}
+}
+
+// Start returns closed channel with a single element to prevent any repeats
+func (s *Once) Start(ctx context.Context) (ch chan struct{}) {
+	ch = make(chan struct{})
+	go func() {
+		ch <- struct{}{}
+		close(ch)
+	}()
+	return ch
+}