From 2add57cfed946b243984f77f8cf91d6a5fa01760 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 19 Sep 2023 09:24:22 -0700 Subject: [PATCH] apply healing per object at 1024 cycles (#18050) - we already have MRF for most recent failures - we trigger healing during HEAD/GET operation These are enough, also change the default max wait from 5sec to 1sec for default scanner speed. --- cmd/data-scanner.go | 10 ++++------ internal/config/scanner/scanner.go | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 2210e157b..abb2469f8 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -54,11 +54,10 @@ const ( dataScannerForceCompactAtFolders = 1_000_000 // Compact when this many subfolders in a single folder (even top level). dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles. - healDeleteDangling = true - healFolderIncludeProb = 32 // Include a clean folder one in n cycles. - healObjectSelectProb = 512 // Overall probability of a file being scanned; one in n. + healDeleteDangling = true + healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n. - dataScannerExcessiveVersionsThreshold = 1000 // Issue a warning when a single object has more versions than this + dataScannerExcessiveVersionsThreshold = 100 // Issue a warning when a single object has more versions than this dataScannerExcessiveFoldersThreshold = 50000 // Issue a warning when a folder has more subfolders than this in a *set* ) @@ -66,7 +65,7 @@ var ( globalHealConfig heal.Config // Sleeper values are updated when config is loaded. - scannerSleeper = newDynamicSleeper(10, 10*time.Second, true) + scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults scannerCycle = uatomic.NewDuration(dataScannerStartDelay) ) @@ -274,7 +273,6 @@ type folderScanner struct { // rarer if the bloom filter for the path is clean and no lifecycles are applied. // Skipped leaves have their totals transferred from the previous cycle. // -// A clean leaf will be included once every healFolderIncludeProb for partial heal scans. // When selected there is a one in healObjectSelectProb that any object will be chosen for heal scan. // // Compaction happens when either: diff --git a/internal/config/scanner/scanner.go b/internal/config/scanner/scanner.go index 77da20675..6f4bc7c0e 100644 --- a/internal/config/scanner/scanner.go +++ b/internal/config/scanner/scanner.go @@ -99,7 +99,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { case "fast": cfg.Delay, cfg.MaxWait, cfg.Cycle = 1, 100*time.Millisecond, time.Minute case "default": - cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, 5*time.Second, time.Minute + cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, time.Second, time.Minute case "slow": cfg.Delay, cfg.MaxWait, cfg.Cycle = 10, 15*time.Second, time.Minute case "slowest":