From 014edd34623376a0705694c4d20de805766692d8 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 30 Mar 2021 13:59:02 -0700 Subject: [PATCH] allow configuring scanner cycles dynamically (#11931) This allows us to speed up or slow down sleeps between multiple scanner cycles, helps in testing as well as some deployments might want to run scanner more frequently. This change is also dynamic can be applied on a running cluster, subsequent cycles pickup the newly set value. --- cmd/config-current.go | 2 ++ cmd/config/scanner/scanner.go | 21 ++++++++++++++++++++- cmd/data-scanner.go | 25 +++++++++++++++++++++---- cmd/logger/target/http/http.go | 12 ++++++------ 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/cmd/config-current.go b/cmd/config-current.go index 1e6f1880e..e6b3ba987 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -624,6 +624,8 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config globalHealConfig = healCfg globalHealConfigMu.Unlock() + // update dynamic scanner values. + scannerCycle.Update(scannerCfg.Cycle) logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait)) // Update all dynamic config values in memory. diff --git a/cmd/config/scanner/scanner.go b/cmd/config/scanner/scanner.go index 9400a3385..ae87d8402 100644 --- a/cmd/config/scanner/scanner.go +++ b/cmd/config/scanner/scanner.go @@ -1,5 +1,5 @@ /* - * MinIO Cloud Storage, (C) 2020 MinIO, Inc. + * MinIO Cloud Storage, (C) 2020-2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,10 @@ import ( const ( Delay = "delay" MaxWait = "max_wait" + Cycle = "cycle" EnvDelay = "MINIO_SCANNER_DELAY" + EnvCycle = "MINIO_SCANNER_CYCLE" EnvDelayLegacy = "MINIO_CRAWLER_DELAY" EnvMaxWait = "MINIO_SCANNER_MAX_WAIT" EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT" @@ -41,6 +43,8 @@ type Config struct { Delay float64 `json:"delay"` // MaxWait is maximum wait time between operations MaxWait time.Duration + // Cycle is the time.Duration between each scanner cycles + Cycle time.Duration } var ( @@ -54,6 +58,10 @@ var ( Key: MaxWait, Value: "15s", }, + config.KV{ + Key: Cycle, + Value: "1m", + }, } // Help provides help for config values @@ -70,6 +78,12 @@ var ( Optional: true, Type: "duration", }, + config.HelpKV{ + Key: Cycle, + Description: `time duration between scanner cycles, defaults to '1m'`, + Optional: true, + Type: "duration", + }, } ) @@ -94,5 +108,10 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { if err != nil { return cfg, err } + + cfg.Cycle, err = time.ParseDuration(env.Get(EnvCycle, kvs.Get(Cycle))) + if err != nil { + return cfg, err + } return cfg, nil } diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 25b5c5cd0..2332c8cc3 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -44,7 +44,6 @@ import ( const ( dataScannerSleepPerFolder = time.Millisecond // Time to wait between folders. - dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles. dataUsageUpdateDirCycles = 16 // Visit all folders every n cycles. healDeleteDangling = true @@ -59,6 +58,7 @@ var ( dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) // Sleeper values are updated when config is loaded. scannerSleeper = newDynamicSleeper(10, 10*time.Second) + scannerCycle = &safeDuration{} ) // initDataScanner will start the scanner in the background. @@ -66,6 +66,23 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) { go runDataScanner(ctx, objAPI) } +type safeDuration struct { + sync.Mutex + t time.Duration +} + +func (s *safeDuration) Update(t time.Duration) { + s.Lock() + defer s.Unlock() + s.t = t +} + +func (s *safeDuration) Get() time.Duration { + s.Lock() + defer s.Unlock() + return s.t +} + // runDataScanner will start a data scanner. // The function will block until the context is canceled. // There should only ever be one scanner running per cluster. @@ -77,7 +94,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { for { ctx, err = locker.GetLock(ctx, dataScannerLeaderLockTimeout) if err != nil { - time.Sleep(time.Duration(r.Float64() * float64(dataScannerStartDelay))) + time.Sleep(time.Duration(r.Float64() * float64(scannerCycle.Get()))) continue } break @@ -101,7 +118,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { br.Close() } - scannerTimer := time.NewTimer(dataScannerStartDelay) + scannerTimer := time.NewTimer(scannerCycle.Get()) defer scannerTimer.Stop() for { @@ -110,7 +127,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) { return case <-scannerTimer.C: // Reset the timer for next cycle. - scannerTimer.Reset(dataScannerStartDelay) + scannerTimer.Reset(scannerCycle.Get()) if intDataUpdateTracker.debug { console.Debugln("starting scanner cycle") diff --git a/cmd/logger/target/http/http.go b/cmd/logger/target/http/http.go index fbdde3deb..2d9767161 100644 --- a/cmd/logger/target/http/http.go +++ b/cmd/logger/target/http/http.go @@ -130,8 +130,8 @@ func (h *Target) startHTTPLogger() { resp, err := h.client.Do(req) cancel() if err != nil { - logger.LogIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration\n", - h.endpoint, err)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration", + h.endpoint, err), h.endpoint) continue } @@ -141,11 +141,11 @@ func (h *Target) startHTTPLogger() { if resp.StatusCode != http.StatusOK { switch resp.StatusCode { case http.StatusForbidden: - logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", - h.endpoint, resp.Status)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", + h.endpoint, resp.Status), h.endpoint) default: - logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration", - h.endpoint, resp.Status)) + logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration", + h.endpoint, resp.Status), h.endpoint) } } }