mirrors/remark42
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update go to 1.20, bump deps

Browse Source
This commit is contained in:
Umputun
2023-08-07 13:03:16 -05:00
parent 7bc7703dc2
commit 2093f4ece2
582 changed files with 43315 additions and 12523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/vendor/github.com/go-pkgz/expirable-cache/.golangci.yml
Generated Vendored
+2 -8
View File
@@ -1,8 +1,6 @@
linters-settings:
govet:
check-shadowing: true
golint:
min-confidence: 0
gocyclo:
min-complexity: 15
maligned:
@@ -25,7 +23,7 @@ linters-settings:
linters:
enable:
- megacheck
- golint
- revive
- govet
- unconvert
- megacheck
@@ -42,7 +40,7 @@ linters:
- varcheck
- stylecheck
- gochecknoinits
- scopelint
- exportloopref
- gocritic
- nakedret
- gosimple
@@ -57,8 +55,4 @@ run:
- vendor
issues:
exclude-rules:
- text: "should have a package comment, unless it's in another file for this package"
linters:
- golint
exclude-use-default: false
backend/vendor/github.com/go-pkgz/expirable-cache/README.md
Generated Vendored
+2 -2
View File
@@ -29,12 +29,12 @@ import (
"fmt"
"time"
"github.com/go-pkgz/expirable-cache"
"github.com/go-pkgz/expirable-cache/v2"
)
func main() {
// make cache with short TTL and 3 max keys
c, _ := cache.NewCache(cache.MaxKeys(3), cache.TTL(time.Millisecond*10))
c := cache.NewCache[string, string]().WithMaxKeys(3).WithTTL(time.Millisecond * 10)
// set value under key1.
// with 0 ttl (last parameter) will use cache-wide setting instead (10ms).
backend/vendor/github.com/go-pkgz/syncs/.golangci.yml
Generated Vendored
+27 -13
View File
@@ -2,7 +2,7 @@ linters-settings:
govet:
check-shadowing: true
golint:
min-confidence: 0
min-confidence: 0.6
gocyclo:
min-complexity: 15
maligned:
@@ -23,47 +23,61 @@ linters-settings:
- experimental
disabled-checks:
- wrapperFunc
- hugeParam
- rangeValCopy
linters:
disable-all: true
enable:
- megacheck
- revive
- govet
- unconvert
- megacheck
- structcheck
- gas
- gocyclo
- dupl
- misspell
- unparam
- varcheck
- deadcode
- unused
- typecheck
- ineffassign
- varcheck
- stylecheck
- gochecknoinits
- exportloopref
- gocritic
- nakedret
- gosimple
- prealloc
fast: false
disable-all: true
run:
output:
format: tab
# modules-download-mode: vendor
skip-dirs:
- vendor
concurrency: 4
issues:
exclude-rules:
- text: "should have a package comment, unless it's in another file for this package"
linters:
- golint
- text: "exitAfterDefer:"
linters:
- gocritic
- text: "whyNoLint: include an explanation for nolint directive"
linters:
- gocritic
- text: "go.mongodb.org/mongo-driver/bson/primitive.E"
linters:
- govet
- text: "weak cryptographic primitive"
linters:
- gosec
- text: "at least one file in a package should have a package comment"
linters:
- stylecheck
- text: "should have a package comment"
linters:
- revive
- text: 'Deferring unsafe method "Close" on type "io.ReadCloser"'
linters:
- gosec
exclude-use-default: false
backend/vendor/github.com/go-pkgz/syncs/README.md
Generated Vendored
+50 -35
View File
@@ -2,7 +2,7 @@
[![Build Status](https://github.com/go-pkgz/syncs/workflows/build/badge.svg)](https://github.com/go-pkgz/syncs/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/go-pkgz/syncs)](https://goreportcard.com/report/github.com/go-pkgz/syncs) [![Coverage Status](https://coveralls.io/repos/github/go-pkgz/syncs/badge.svg?branch=master)](https://coveralls.io/github/go-pkgz/syncs?branch=master)
Package syncs provides additional synchronization primitives.
The `syncs` package offers extra synchronization primitives, such as `Semaphore`, `SizedGroup`, and `ErrSizedGroup`, to help manage concurrency in Go programs. With `syncs` package, you can efficiently manage concurrency in your Go programs using additional synchronization primitives. Use them according to your specific use-case requirements to control and limit concurrent goroutines while handling errors and early termination effectively.
## Install and update
@@ -12,56 +12,71 @@ Package syncs provides additional synchronization primitives.
### Semaphore
Implements `sync.Locker` interface but for given capacity, thread safe. Lock increases count and Unlock - decreases. Unlock on 0 count will be blocked.
`Semaphore` implements the `sync.Locker` interface with an additional `TryLock` function and a specified capacity.
It is thread-safe. The `Lock` function increases the count, while Unlock decreases it. When the count is 0, `Unlock` will block, and `Lock` will block until the count is greater than 0. The `TryLock` function will return false if locking failed (i.e. semaphore is locked) and true otherwise.
```go
sema := syncs.NewSemaphore(10) // make semaphore with 10 initial capacity
for i :=0; i<10; i++ {
sema.Lock() // all 10 locks will pass, i.w. won't lock
}
sema.Lock() // this is 11 - will lock for real
sema := syncs.NewSemaphore(10) // make semaphore with 10 initial capacity
for i :=0; i<10; i++ {
sema.Lock() // all 10 locks will pass, i.w. won't lock
}
sema.Lock() // this is 11 - will lock for real
// in some other place/goroutine
sema.Unlock() // decrease semaphore counter
// in some other place/goroutine
sema.Unlock() // decrease semaphore counter
ok := sema.TryLock() // try to lock, will return false if semaphore is locked
```
### SizedGroup
Mix semaphore and WaitGroup to provide sized waiting group. The result is a wait group allowing limited number of goroutine to run in parallel.
`SizedGroup` combines `Semaphore` and `WaitGroup` to provide a wait group that allows a limited number of goroutines to run in parallel.
By default, locking happens inside the goroutine. This means every call will be non-blocking, but some goroutines may wait if the semaphore is locked. Technically, it doesn't limit the number of goroutines but rather the number of running (active) goroutines.
To block goroutines from starting, use the `Preemptive` option. Important: With `Preemptive`, the `Go` call can block. If the maximum size is reached, the call will wait until the number of running goroutines drops below the maximum. This not only limits the number of running goroutines but also the number of waiting goroutines.
By default, the locking happens inside of goroutine, i.e. **every call will be non-blocked**, but some goroutines may wait if semaphore locked. It means - technically it doesn't limit number of goroutines, but rather number of running (active) goroutines.
In order to block goroutines from even starting use `Preemptive` option (see below).
```go
swg := syncs.NewSizedGroup(5) // wait group with max size=5
for i :=0; i<10; i++ {
swg.Go(fn func(ctx context.Context){
doThings(ctx) // only 5 of these will run in parallel
})
}
swg.Wait()
swg := syncs.NewSizedGroup(5) // wait group with max size=5
for i :=0; i<10; i++ {
swg.Go(func(ctx context.Context){
doThings(ctx) // only 5 of these will run in parallel
})
}
swg.Wait()
```
Another option is `Discard`, which will skip (won't start) goroutines if the semaphore is locked. In other words, if a defined number of goroutines are already running, the call will be discarded. `Discard` is useful when you don't care about the results of extra goroutines; i.e., you just want to run some tasks in parallel but can allow some number of them to be ignored. This flag sets `Preemptive` as well, because otherwise, it doesn't make sense.
```go
swg := syncs.NewSizedGroup(5, Discard) // wait group with max size=5 and discarding extra goroutines
for i :=0; i<10; i++ {
swg.Go(func(ctx context.Context){
doThings(ctx) // only 5 of these will run in parallel and 5 other can be discarded
})
}
swg.Wait()
```
### ErrSizedGroup
Sized error group is a SizedGroup with error control.
Works the same as errgrp.Group, i.e. returns first error.
Can work as regular errgrp.Group or with early termination.
Thread safe.
`ErrSizedGroup` is a `SizedGroup` with error control. It works the same as `errgrp.Group`, i.e., it returns the first error.
It can work as a regular errgrp.Group or with early termination. It is thread-safe.
Supports both in-goroutine-wait via `NewErrSizedGroup` as well as outside of goroutine wait with `Preemptive` option. Another options are `TermOnErr` which will skip (won't start) all other goroutines if any error returned, and `Context` for early termination/timeouts.
Important! With `Preemptive` Go call **can block**. In case if maximum size reached the call will wait till number of running goroutines
dropped under max. This way we not only limiting number of running goroutines but also number of waiting goroutines.
`ErrSizedGroup` supports both in-goroutine-wait as well as outside of goroutine wait with `Preemptive` and `Discard` options (see above). Other options include `TermOnErr`, which skips (won't start) all other goroutines if any error is returned, and `Context` for early termination/timeouts.
```go
ewg := syncs.NewErrSizedGroup(5, syncs.Preemptive) // error wait group with max size=5, don't try to start more if any error happened
for i :=0; i<10; i++ {
ewg.Go(fn func(ctx context.Context) error { // Go here could be blocked if trying to run >5 at the same time
err := doThings(ctx) // only 5 of these will run in parallel
return err
})
}
err := ewg.Wait()
```
ewg := syncs.NewErrSizedGroup(5, syncs.Preemptive) // error wait group with max size=5, don't try to start more if any error happened
for i :=0; i<10; i++ {
ewg.Go(func(ctx context.Context) error { // Go here could be blocked if trying to run >5 at the same time
err := doThings(ctx) // only 5 of these will run in parallel
return err
})
}
err := ewg.Wait()
```
backend/vendor/github.com/go-pkgz/syncs/errsizedgroup.go
Generated Vendored
+61 -24
View File
@@ -12,21 +12,21 @@ import (
type ErrSizedGroup struct {
options
wg sync.WaitGroup
sema sync.Locker
sema Locker
err *multierror
err *MultiError
errLock sync.RWMutex
errOnce sync.Once
}
// NewErrSizedGroup makes wait group with limited size alive goroutines.
// By default all goroutines will be started but will wait inside. For limited number of goroutines use Preemptive() options.
// By default, all goroutines will be started but will wait inside.
// For limited number of goroutines use Preemptive() options.
// TermOnErr will skip (won't start) all other goroutines if any error returned.
func NewErrSizedGroup(size int, options ...GroupOption) *ErrSizedGroup {
res := ErrSizedGroup{
sema: NewSemaphore(size),
err: new(multierror),
err: new(MultiError),
}
for _, opt := range options {
@@ -41,10 +41,43 @@ func NewErrSizedGroup(size int, options ...GroupOption) *ErrSizedGroup {
// returned by Wait. If no termOnError all errors will be collected in multierror.
func (g *ErrSizedGroup) Go(f func() error) {
canceled := func() bool {
if g.ctx == nil {
return false
}
select {
case <-g.ctx.Done():
return true
default:
return false
}
}
if canceled() {
g.errOnce.Do(func() {
// don't repeat this error
g.err.append(g.ctx.Err())
})
return
}
g.wg.Add(1)
isLocked := false
if g.preLock {
g.sema.Lock()
lockOk := g.sema.TryLock()
if lockOk {
isLocked = true
}
if !lockOk && g.discardIfFull {
// lock failed and discardIfFull is set, discard this goroutine
g.wg.Done()
return
}
if !lockOk && !g.discardIfFull {
g.sema.Lock() // make sure we have block until lock is acquired
isLocked = true
}
}
go func() {
@@ -57,30 +90,29 @@ func (g *ErrSizedGroup) Go(f func() error) {
}
g.errLock.RLock()
defer g.errLock.RUnlock()
return g.err.errorOrNil() != nil
return g.err.ErrorOrNil() != nil
}
defer func() {
if isLocked {
g.sema.Unlock()
}
}()
if terminated() {
return // terminated due prev error, don't run anything in this group anymore
}
if !g.preLock {
g.sema.Lock()
isLocked = true
}
if err := f(); err != nil {
g.errLock.Lock()
g.err = g.err.append(err)
g.errLock.Unlock()
g.errOnce.Do(func() { // call context cancel once
if g.cancel != nil {
g.cancel()
}
})
}
g.sema.Unlock()
}()
}
@@ -88,25 +120,24 @@ func (g *ErrSizedGroup) Go(f func() error) {
// returns all errors (if any) wrapped with multierror from them.
func (g *ErrSizedGroup) Wait() error {
g.wg.Wait()
if g.cancel != nil {
g.cancel()
}
return g.err.errorOrNil()
return g.err.ErrorOrNil()
}
type multierror struct {
// MultiError is a thread safe container for multi-error type that implements error interface
type MultiError struct {
errors []error
lock sync.Mutex
}
func (m *multierror) append(err error) *multierror {
func (m *MultiError) append(err error) *MultiError {
m.lock.Lock()
m.errors = append(m.errors, err)
m.lock.Unlock()
return m
}
func (m *multierror) errorOrNil() error {
// ErrorOrNil returns nil if no errors or multierror if errors occurred
func (m *MultiError) ErrorOrNil() error {
m.lock.Lock()
defer m.lock.Unlock()
if len(m.errors) == 0 {
@@ -115,8 +146,8 @@ func (m *multierror) errorOrNil() error {
return m
}
// Error returns multierror string
func (m *multierror) Error() string {
// Error returns multi-error string
func (m *MultiError) Error() string {
m.lock.Lock()
defer m.lock.Unlock()
if len(m.errors) == 0 {
@@ -130,3 +161,9 @@ func (m *multierror) Error() string {
}
return fmt.Sprintf("%d error(s) occurred: %s", len(m.errors), strings.Join(errs, ", "))
}
func (m *MultiError) Errors() []error {
m.lock.Lock()
defer m.lock.Unlock()
return m.errors
}
backend/vendor/github.com/go-pkgz/syncs/group_options.go
Generated Vendored
+12 -6
View File
@@ -3,19 +3,19 @@ package syncs
import "context"
type options struct {
ctx context.Context
cancel context.CancelFunc
preLock bool
termOnError bool
ctx context.Context
preLock bool
termOnError bool
discardIfFull bool
}
// GroupOption functional option type
type GroupOption func(o *options)
// Context passes ctx and makes it cancelable
// Context passes ctx to group, goroutines will be canceled if ctx is canceled
func Context(ctx context.Context) GroupOption {
return func(o *options) {
o.ctx, o.cancel = context.WithCancel(ctx)
o.ctx = ctx
}
}
@@ -28,3 +28,9 @@ func Preemptive(o *options) {
func TermOnErr(o *options) {
o.termOnError = true
}
// Discard will discard new goroutines if semaphore is full, i.e. no more goroutines allowed
func Discard(o *options) {
o.discardIfFull = true
o.preLock = true // discard implies preemptive
}
backend/vendor/github.com/go-pkgz/syncs/semaphore.go
Generated Vendored
+18 -2
View File
@@ -2,14 +2,20 @@ package syncs
import "sync"
// Locker is a superset of sync.Locker interface with TryLock method.
type Locker interface {
sync.Locker
TryLock() bool
}
// Semaphore implementation, counted lock only. Implements sync.Locker interface, thread safe.
type semaphore struct {
sync.Locker
Locker
ch chan struct{}
}
// NewSemaphore makes Semaphore with given capacity
func NewSemaphore(capacity int) sync.Locker {
func NewSemaphore(capacity int) Locker {
if capacity <= 0 {
capacity = 1
}
@@ -25,3 +31,13 @@ func (s *semaphore) Lock() {
func (s *semaphore) Unlock() {
<-s.ch
}
// TryLock acquires semaphore if possible, returns true if acquired, false otherwise.
func (s *semaphore) TryLock() bool {
select {
case s.ch <- struct{}{}:
return true
default:
return false
}
}
backend/vendor/github.com/go-pkgz/syncs/sizedgroup.go
Generated Vendored
+10 -5
View File
@@ -11,7 +11,7 @@ import (
type SizedGroup struct {
options
wg sync.WaitGroup
sema sync.Locker
sema Locker
}
// NewSizedGroup makes wait group with limited size alive goroutines
@@ -27,7 +27,6 @@ func NewSizedGroup(size int, opts ...GroupOption) *SizedGroup {
// Go calls the given function in a new goroutine.
// Every call will be unblocked, but some goroutines may wait if semaphore locked.
func (g *SizedGroup) Go(fn func(ctx context.Context)) {
canceled := func() bool {
select {
case <-g.ctx.Done():
@@ -41,12 +40,18 @@ func (g *SizedGroup) Go(fn func(ctx context.Context)) {
return
}
g.wg.Add(1)
if g.preLock {
g.sema.Lock()
lockOk := g.sema.TryLock()
if !lockOk && g.discardIfFull {
// lock failed and discardIfFull is set, discard this goroutine
return
}
if !lockOk && !g.discardIfFull {
g.sema.Lock() // make sure we have block until lock is acquired
}
}
g.wg.Add(1)
go func() {
defer g.wg.Done()