mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-23 16:43:23 +00:00
Chris LuandGitHub
298ab35fd7
* shell: ec.encode batches 10 volumes by default Encoding a whole collection as one batch means a late failure leaves everything half-converted. Default -batchSize to 10 so each batch is encoded, rebalanced, verified, and its originals deleted before the next starts. -batchSize=0 keeps the all-at-once behavior. Batch progress messages now print only when there is more than one batch, so small runs read as before. * shell: ec.decode decodes in batches, refreshing topology in between ec.decode walked every volume off the single topology snapshot taken at startup, which goes stale as earlier decodes move shards around and create volumes. Decode 10 volumes per batch by default, re-collecting the topology and rebuilding the free-space accounting between batches. -batchSize=0 keeps the single-snapshot behavior.
85 lines
2.2 KiB
Go
85 lines
2.2 KiB
Go
package shell
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
)
|
|
|
|
var (
|
|
// Default maximum parallelization/concurrency for commands supporting it.
|
|
DefaultMaxParallelization = 10
|
|
// Default number of volumes EC encode/decode process per batch.
|
|
DefaultEcBatchSize = 10
|
|
// CollectionDefault is the special keyword to match empty collection names.
|
|
// Use "_default" to avoid collision with a literal collection named "default".
|
|
CollectionDefault = "_default"
|
|
)
|
|
|
|
// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any.
|
|
type ErrorWaitGroup struct {
|
|
maxConcurrency int
|
|
wg *sync.WaitGroup
|
|
wgSem chan bool
|
|
errors []error
|
|
errorsMu sync.Mutex
|
|
}
|
|
|
|
type ErrorWaitGroupTask func() error
|
|
|
|
func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup {
|
|
if maxConcurrency <= 0 {
|
|
// no concurrency = one task at the time
|
|
maxConcurrency = 1
|
|
}
|
|
return &ErrorWaitGroup{
|
|
maxConcurrency: maxConcurrency,
|
|
wg: &sync.WaitGroup{},
|
|
wgSem: make(chan bool, maxConcurrency),
|
|
}
|
|
}
|
|
|
|
// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed.
|
|
func (ewg *ErrorWaitGroup) Reset() {
|
|
close(ewg.wgSem)
|
|
|
|
ewg.wg = &sync.WaitGroup{}
|
|
ewg.wgSem = make(chan bool, ewg.maxConcurrency)
|
|
ewg.errors = nil
|
|
}
|
|
|
|
// Add queues an ErrorWaitGroupTask to be executed as a goroutine.
|
|
func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) {
|
|
if ewg.maxConcurrency <= 1 {
|
|
// keep run order deterministic when parallelization is off
|
|
ewg.errors = append(ewg.errors, f())
|
|
return
|
|
}
|
|
|
|
ewg.wg.Add(1)
|
|
go func() {
|
|
ewg.wgSem <- true
|
|
|
|
err := f()
|
|
ewg.errorsMu.Lock()
|
|
ewg.errors = append(ewg.errors, err)
|
|
ewg.errorsMu.Unlock()
|
|
|
|
<-ewg.wgSem
|
|
ewg.wg.Done()
|
|
}()
|
|
}
|
|
|
|
// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines.
|
|
func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) {
|
|
ewg.errorsMu.Lock()
|
|
ewg.errors = append(ewg.errors, fmt.Errorf(format, a...))
|
|
ewg.errorsMu.Unlock()
|
|
}
|
|
|
|
// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them.
|
|
func (ewg *ErrorWaitGroup) Wait() error {
|
|
ewg.wg.Wait()
|
|
return errors.Join(ewg.errors...)
|
|
}
|