From a406bb02885408421547c9f6a845a146c3b9d82d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 8 Aug 2022 16:16:44 -0700 Subject: [PATCH] restrict number of disks used for scanning buckets upto GOMAXPROCS (#15492) control scanner parallelism to avoid higher CPU usage on nodes that have more drives but an old CPU. --- cmd/erasure.go | 8 ++++++++ internal/config/scanner/scanner.go | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/erasure.go b/cmd/erasure.go index 840128caa..9697fbddf 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -23,6 +23,7 @@ import ( "fmt" "math/rand" "os" + "runtime" "sort" "sync" "time" @@ -439,6 +440,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf r := rand.New(rand.NewSource(time.Now().UnixNano())) r.Shuffle(len(disks), func(i, j int) { disks[i], disks[j] = disks[j], disks[i] }) + // Restrict parallelism for disk usage scanner + // upto GOMAXPROCS if GOMAXPROCS is < len(disks) + maxProcs := runtime.GOMAXPROCS(0) + if maxProcs < len(disks) { + disks = disks[:maxProcs] + } + // Start one scanner per disk var wg sync.WaitGroup wg.Add(len(disks)) diff --git a/internal/config/scanner/scanner.go b/internal/config/scanner/scanner.go index 56fa49e1c..f4cc97085 100644 --- a/internal/config/scanner/scanner.go +++ b/internal/config/scanner/scanner.go @@ -52,11 +52,11 @@ type Config struct { var DefaultKVS = config.KVS{ config.KV{ Key: Delay, - Value: "10", + Value: "2", }, config.KV{ Key: MaxWait, - Value: "15s", + Value: "5s", }, config.KV{ Key: Cycle,