admin: address review feedback on the maintenance scanner fix

This commit is contained in:
Chris Lu
2026-08-23 00:26:38 -07:00
parent 8d8a25b1cf
commit 3b8931c2f6
9 changed files with 28 additions and 25 deletions
@@ -7,8 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
)
// TestLoadMaintenanceConfigHonoursPersistedTaskConfigs guards against the regression in
// https://github.com/seaweedfs/seaweedfs/issues/10874: buildPolicyFromTaskConfigs used to call
// TestLoadMaintenanceConfigHonoursPersistedTaskConfigs guards against a regression: buildPolicyFromTaskConfigs used to call
// LoadConfigFromPersistence(nil), which can never satisfy the loaders' type assertion, so every
// task silently fell back to its compiled-in defaults (Enabled: true) and a task disabled in the
// admin UI kept being scheduled.
+1 -1
View File
@@ -25,7 +25,7 @@ func restoreGlobalTaskState(t *testing.T) {
// data directory that has a disabled balance task saved in it, and checks the end state that
// actually matters: the balance detector reports disabled, so ScanWithTaskDetectors skips it.
//
// This is the whole of issue #10874 in one test. The reporter disabled balance, and the
// This is the whole reported bug in one test. The reporter disabled balance, and the
// scanner kept detecting balance tasks, cancelling them and re-detecting them. Two separate
// defects had to line up for the disabled flag to survive to here: the policy had to be built
// from the persisted configs rather than from a nil store, and the policy had to reach
+11 -5
View File
@@ -129,9 +129,15 @@ func TestEcBalanceTaskPolicyRoundTrip(t *testing.T) {
t.Errorf("ec_balance collection filter = %q, want the persisted %q", loaded.CollectionFilter, "pictures")
}
// The generic dispatcher the maintenance manager uses has to know the type too.
if err := cp.SaveTaskPolicy("ec_balance", saved.ToTaskPolicy()); err != nil {
t.Errorf("SaveTaskPolicy(ec_balance): %v", err)
// The generic dispatcher the maintenance manager uses has to know the type too, and
// has to write the same file the dedicated loader reads.
dispatched := NewConfigPersistence(t.TempDir())
if err := dispatched.SaveTaskPolicy("ec_balance", saved.ToTaskPolicy()); err != nil {
t.Fatalf("SaveTaskPolicy(ec_balance): %v", err)
}
roundTripped := ec_balance.LoadConfigFromPersistence(dispatched)
if roundTripped == nil || roundTripped.MinServerCount != 9 {
t.Errorf("SaveTaskPolicy(ec_balance) did not round-trip through the ec_balance store: %+v", roundTripped)
}
}
@@ -164,8 +170,8 @@ func TestBuildPolicyKeepsTaskSpecificFields(t *testing.T) {
if got := ecPolicy.GetReplicaPlacement(); got != "020" {
t.Errorf("erasure coding replica placement = %q, want the persisted %q", got, "020")
}
if got := ecPolicy.GetPreferredTags(); len(got) != 2 {
t.Errorf("erasure coding preferred tags = %v, want the persisted 2 entries", got)
if got := ecPolicy.GetPreferredTags(); len(got) != 2 || got[0] != "ssd" || got[1] != "archive" {
t.Errorf("erasure coding preferred tags = %v, want the persisted [ssd archive]", got)
}
balancePolicy := policy.TaskPolicies["balance"].GetBalanceConfig()
@@ -201,8 +201,7 @@ func (mm *MaintenanceManager) scanLoop() {
// configured interval instead never restores the normal cadence: once the errors stop,
// getScanInterval returns scanInterval again, the comparison comes out false, and the
// ticker is left at the backoff delay. A single transient scan failure therefore pinned
// the scanner to one scan per second forever - see issue #10874, where that produced a
// ~658 KB/s log flood and 193k orphaned task files.
// the scanner to one scan per second forever.
activeInterval := scanInterval
ticker := time.NewTicker(activeInterval)
// Wrapped in a closure so the replacement ticker is stopped, not the one that happened
@@ -70,8 +70,7 @@ func disabledStub() *stubConfigPersistence {
}
}
// TestBuildPolicyFromTaskConfigsUsesPersistence covers the bug reported in
// https://github.com/seaweedfs/seaweedfs/issues/10874: the persistence argument used to be a
// TestBuildPolicyFromTaskConfigsUsesPersistence covers a reported bug: the persistence argument used to be a
// literal nil, which no type assertion can satisfy, so a task disabled on disk came back enabled.
func TestBuildPolicyFromTaskConfigsUsesPersistence(t *testing.T) {
policy := BuildPolicyFromTaskConfigs(disabledStub())
@@ -38,7 +38,7 @@ func snapshotDetectorState(t *testing.T) {
})
}
// TestPolicyReachesRegisteredDetectors is the regression test for the half of issue #10874
// TestPolicyReachesRegisteredDetectors is the regression test for the half of the scan-flood bug
// that a corrected policy alone did not fix: MaintenanceIntegration pushes the policy into
// detectors and schedulers through interface{ SetEnabled(bool) } and
// interface{ SetMaxConcurrent(int) } type assertions, but every task is backed by
@@ -39,8 +39,7 @@ func (c *scanCadenceClient) callCount() int {
return c.calls
}
// TestScanLoopRestoresIntervalAfterBackoff is the regression test for the scan flood in
// https://github.com/seaweedfs/seaweedfs/issues/10874. The loop shortens its ticker to the
// TestScanLoopRestoresIntervalAfterBackoff is the regression test for the scan flood. The loop shortens its ticker to the
// error backoff delay after a failed scan. It used to compare the target interval against
// the *configured* scan interval rather than against the interval the ticker was actually
// running at, so once the errors stopped the comparison came out false and the ticker was
@@ -104,15 +103,16 @@ func TestScanLoopRestoresIntervalAfterBackoff(t *testing.T) {
// Observe a window that a 1s cadence would fill with scans and a 3s cadence would not.
const window = 5 * time.Second
time.Sleep(window)
scansInWindow := client.callCount() - before
callsInWindow := client.callCount() - before
// 3s cadence: at most 2 scans in 5s. 1s cadence: about 5.
if scansInWindow > 2 {
t.Errorf("scan loop ran %d scans in %v after recovering from a failed scan; "+
// One scan makes two master-client calls. 3s cadence: at most 2 scans (4 calls) in 5s,
// even if a stall shifts the window. 1s cadence: about 5 scans (10 calls).
if callsInWindow > 4 {
t.Errorf("scan loop made %d master-client calls in %v after recovering from a failed scan; "+
"the ticker was left at the %v backoff instead of returning to the configured %v",
scansInWindow, window, time.Second, baseInterval)
callsInWindow, window, time.Second, baseInterval)
}
if scansInWindow == 0 {
if callsInWindow == 0 {
t.Errorf("scan loop ran no scans in %v, expected the %v cadence to fire at least once", window, baseInterval)
}
}
+2 -2
View File
@@ -49,7 +49,7 @@ func (d *GenericDetector) IsEnabled() bool {
// MaintenanceIntegration.configureDetectorFromPolicy). Every registered task is
// backed by this generic detector, so without this method that assertion failed
// for every task and the policy never reached the flag that
// ScanWithTaskDetectors actually gates on. See issue #10874.
// ScanWithTaskDetectors actually gates on.
func (d *GenericDetector) SetEnabled(enabled bool) {
d.taskDef.Config.SetEnabled(enabled)
}
@@ -143,7 +143,7 @@ func (s *GenericScheduler) IsEnabled() bool {
// SetEnabled turns scheduling for this task type on or off. Detector and scheduler
// share one TaskDefinition, so this is the same flag GenericDetector.SetEnabled sets;
// both setters exist because the maintenance integration configures the two
// independently. See GenericDetector.SetEnabled and issue #10874.
// independently. See GenericDetector.SetEnabled.
func (s *GenericScheduler) SetEnabled(enabled bool) {
s.taskDef.Config.SetEnabled(enabled)
}
@@ -18,8 +18,8 @@ func (s *stubVacuumStore) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error)
// wrongShapedStore is what the maintenance manager used to be handed: a non-nil value that
// does not satisfy the accessor the loader asserts on. It has to keep falling back to the
// defaults, but no longer silently - the reporter of issue #10874 had to read the source to
// find out why their disabled task kept running.
// defaults, but no longer silently: a fallback the operator cannot see reads as a disabled task
// that keeps running.
type wrongShapedStore struct{}
func (wrongShapedStore) SomethingElse() {}