From 745e864bdaa4a59973063304cd762c84cde1944c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 May 2026 12:39:37 -0700 Subject: [PATCH] feat(s3/lifecycle): retire algorithm flag, daily_replay is the only path (Phase 5a) (#9465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(s3/lifecycle): remove algorithm flag, daily_replay is the only path (Phase 5a) With Phase 4b on master the daily_replay path covers every rule kind and the streaming algorithm has no remaining responsibilities. This PR retires the algorithm flag from the worker: - Drop the "Algorithm" enum field from AdminConfigForm and its DefaultValues entry. - Drop the if/else routing in Execute — every Execute call now routes straight into executeDailyReplay. - Drop the streaming-only worker fields (DispatchTick, CheckpointTick, RefreshInterval, BootstrapInterval) and their matching form fields. None of them are read by the daily_replay path; keeping them in the form would suggest tuning knobs that don't do anything. - Drop AlgorithmStreaming / AlgorithmDailyReplay constants and the Config.Algorithm field. The streaming-path packages (s3lifecycle/scheduler, s3lifecycle/dispatcher) remain on the tree; they're now reachable only by the weed shell s3.lifecycle.run-shard debug command and the few helpers (LoadCompileInputs, FilerStore, FilerSiblingLister) the daily_replay worker still uses. Phase 5b deletes the dead code. Tests prune the cadence-default assertions to the single remaining field (max_runtime_minutes). --- weed/worker/tasks/s3_lifecycle/config.go | 69 ++------------ weed/worker/tasks/s3_lifecycle/config_test.go | 93 +------------------ weed/worker/tasks/s3_lifecycle/handler.go | 87 ++--------------- .../worker/tasks/s3_lifecycle/handler_test.go | 11 +-- 4 files changed, 20 insertions(+), 240 deletions(-) diff --git a/weed/worker/tasks/s3_lifecycle/config.go b/weed/worker/tasks/s3_lifecycle/config.go index ed3ec9f8c..5831e3581 100644 --- a/weed/worker/tasks/s3_lifecycle/config.go +++ b/weed/worker/tasks/s3_lifecycle/config.go @@ -9,80 +9,25 @@ import ( const ( jobType = "s3_lifecycle" - // shardPipelineGoroutines is the in-process fan-out across the - // 16-shard space. Kept as a hardcoded internal default — formerly - // an admin form field, removed because it's a per-worker tuning - // knob, not a cluster-coordination concern. + // In-process fan-out across the 16 shards. shardPipelineGoroutines = 1 - defaultDispatchTickMinutes = int64(1) - defaultCheckpointTickSeconds = int64(30) - defaultRefreshIntervalMinutes = int64(5) - defaultMaxRuntimeMinutes = int64(60) - defaultBootstrapIntervalMinutes = int64(0) // 0 = walk once per process - - // AlgorithmDailyReplay routes the worker through dailyrun.Run for - // one bounded pass per Execute. Currently Phase 2 / replay-only: - // buckets with walker-bound action kinds are refused. Phase 4 - // extends this to handle every kind. Default — the streaming path - // stays in the tree as a runtime escape hatch only. - AlgorithmDailyReplay = "daily_replay" - // AlgorithmStreaming is the legacy event-driven dispatcher path - // (reader + heap + per-shard pipeline). Kept as a fallback knob for - // rollout; deleted by Phase 5 once Phase 4 walker integration ships. - AlgorithmStreaming = "streaming" - - defaultAlgorithm = AlgorithmDailyReplay + defaultMaxRuntimeMinutes = int64(60) ) -// Config is the parsed AdminConfigForm + WorkerConfigForm view. type Config struct { - Workers int - DispatchTick time.Duration - CheckpointTick time.Duration - RefreshInterval time.Duration - BootstrapInterval time.Duration - MaxRuntime time.Duration - Algorithm string + Workers int + MaxRuntime time.Duration } -// ParseConfig pulls the lifecycle Handler config from the merged -// admin+worker config values. Missing fields fall back to defaults. -func ParseConfig(adminValues, workerValues map[string]*plugin_pb.ConfigValue) Config { +func ParseConfig(_ map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config { cfg := Config{ - Workers: shardPipelineGoroutines, - DispatchTick: time.Duration(readInt64(workerValues, "dispatch_tick_minutes", defaultDispatchTickMinutes)) * time.Minute, - CheckpointTick: time.Duration(readInt64(workerValues, "checkpoint_tick_seconds", defaultCheckpointTickSeconds)) * time.Second, - RefreshInterval: time.Duration(readInt64(workerValues, "refresh_interval_minutes", defaultRefreshIntervalMinutes)) * time.Minute, - BootstrapInterval: time.Duration(readInt64(workerValues, "bootstrap_interval_minutes", defaultBootstrapIntervalMinutes)) * time.Minute, - MaxRuntime: time.Duration(readInt64(workerValues, "max_runtime_minutes", defaultMaxRuntimeMinutes)) * time.Minute, - Algorithm: readString(adminValues, "algorithm", defaultAlgorithm), - } - if cfg.DispatchTick <= 0 { - cfg.DispatchTick = time.Duration(defaultDispatchTickMinutes) * time.Minute - } - if cfg.CheckpointTick <= 0 { - cfg.CheckpointTick = time.Duration(defaultCheckpointTickSeconds) * time.Second - } - if cfg.RefreshInterval <= 0 { - cfg.RefreshInterval = time.Duration(defaultRefreshIntervalMinutes) * time.Minute - } - // BootstrapInterval is intentionally NOT clamped — zero means - // "walk once per process", which is the legacy default for any - // deployment that hasn't opted into a cadence yet. Negative values - // fall back to zero. - if cfg.BootstrapInterval < 0 { - cfg.BootstrapInterval = 0 + Workers: shardPipelineGoroutines, + MaxRuntime: time.Duration(readInt64(workerValues, "max_runtime_minutes", defaultMaxRuntimeMinutes)) * time.Minute, } if cfg.MaxRuntime <= 0 { cfg.MaxRuntime = time.Duration(defaultMaxRuntimeMinutes) * time.Minute } - switch cfg.Algorithm { - case AlgorithmStreaming, AlgorithmDailyReplay: - // valid - default: - cfg.Algorithm = defaultAlgorithm - } return cfg } diff --git a/weed/worker/tasks/s3_lifecycle/config_test.go b/weed/worker/tasks/s3_lifecycle/config_test.go index dc7b91c7d..4e5d919aa 100644 --- a/weed/worker/tasks/s3_lifecycle/config_test.go +++ b/weed/worker/tasks/s3_lifecycle/config_test.go @@ -12,112 +12,27 @@ func TestParseConfigDefaults(t *testing.T) { if cfg.Workers != shardPipelineGoroutines { t.Errorf("Workers default=%d, want %d", cfg.Workers, shardPipelineGoroutines) } - if cfg.DispatchTick != 1*time.Minute { - t.Errorf("DispatchTick default=%v, want 1m", cfg.DispatchTick) - } - if cfg.CheckpointTick != 30*time.Second { - t.Errorf("CheckpointTick default=%v, want 30s", cfg.CheckpointTick) - } - if cfg.RefreshInterval != 5*time.Minute { - t.Errorf("RefreshInterval default=%v, want 5m", cfg.RefreshInterval) - } if cfg.MaxRuntime != 60*time.Minute { t.Errorf("MaxRuntime default=%v, want 60m", cfg.MaxRuntime) } - if cfg.BootstrapInterval != 0 { - t.Errorf("BootstrapInterval default=%v, want 0 (walk-once-per-process)", cfg.BootstrapInterval) - } - if cfg.Algorithm != AlgorithmDailyReplay { - t.Errorf("Algorithm default=%q, want %q", cfg.Algorithm, AlgorithmDailyReplay) - } } -func TestParseConfig_AlgorithmStreamingExplicit(t *testing.T) { - // Streaming stays available as a rollout escape hatch until Phase 5 - // deletes it. Operators must be able to opt back into it explicitly. - admin := map[string]*plugin_pb.ConfigValue{ - "algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: AlgorithmStreaming}}, - } - cfg := ParseConfig(admin, nil) - if cfg.Algorithm != AlgorithmStreaming { - t.Errorf("Algorithm=%q, want %q", cfg.Algorithm, AlgorithmStreaming) - } -} - -func TestParseConfig_AlgorithmUnknownValueFallsBackToDefault(t *testing.T) { - // Operators should not be able to silently activate a future - // algorithm value by typo. Anything not in the enum falls back to - // the default (daily_replay). - admin := map[string]*plugin_pb.ConfigValue{ - "algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "future_algo"}}, - } - cfg := ParseConfig(admin, nil) - if cfg.Algorithm != AlgorithmDailyReplay { - t.Errorf("unknown algorithm=%q must fall back to %q, got %q", "future_algo", AlgorithmDailyReplay, cfg.Algorithm) - } -} - -func TestParseConfigOverrides(t *testing.T) { +func TestParseConfigOverrideMaxRuntime(t *testing.T) { worker := map[string]*plugin_pb.ConfigValue{ - "dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2}}, - "checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 15}}, - "refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10}}, - "bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}}, - "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}}, + "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}}, } cfg := ParseConfig(nil, worker) - if cfg.DispatchTick != 2*time.Minute { - t.Errorf("DispatchTick=%v, want 2m", cfg.DispatchTick) - } - if cfg.CheckpointTick != 15*time.Second { - t.Errorf("CheckpointTick=%v, want 15s", cfg.CheckpointTick) - } - if cfg.RefreshInterval != 10*time.Minute { - t.Errorf("RefreshInterval=%v, want 10m", cfg.RefreshInterval) - } - if cfg.BootstrapInterval != 120*time.Minute { - t.Errorf("BootstrapInterval=%v, want 120m", cfg.BootstrapInterval) - } if cfg.MaxRuntime != 120*time.Minute { t.Errorf("MaxRuntime=%v, want 120m", cfg.MaxRuntime) } } -func TestParseConfigClampsZeroAndNegative(t *testing.T) { +func TestParseConfigNegativeMaxRuntimeClampsToDefault(t *testing.T) { worker := map[string]*plugin_pb.ConfigValue{ - "dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, - "checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, - "refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, - "bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}}, - "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}}, + "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}}, } cfg := ParseConfig(nil, worker) - if cfg.DispatchTick != 1*time.Minute { - t.Errorf("zero DispatchTick should clamp to default, got %v", cfg.DispatchTick) - } - if cfg.CheckpointTick != 30*time.Second { - t.Errorf("zero CheckpointTick should clamp to default, got %v", cfg.CheckpointTick) - } - if cfg.RefreshInterval != 5*time.Minute { - t.Errorf("zero RefreshInterval should clamp to default, got %v", cfg.RefreshInterval) - } - if cfg.BootstrapInterval != 0 { - t.Errorf("negative BootstrapInterval should clamp to 0, got %v", cfg.BootstrapInterval) - } if cfg.MaxRuntime != 60*time.Minute { t.Errorf("negative MaxRuntime should clamp to default, got %v", cfg.MaxRuntime) } } - -func TestParseConfigBootstrapIntervalZeroIsLegacy(t *testing.T) { - // Explicit zero stays zero (walk-once-per-process). Existing - // deployments that don't set bootstrap_interval_minutes get the - // legacy behavior unchanged. - worker := map[string]*plugin_pb.ConfigValue{ - "bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, - } - cfg := ParseConfig(nil, worker) - if cfg.BootstrapInterval != 0 { - t.Errorf("zero BootstrapInterval must stay zero, got %v", cfg.BootstrapInterval) - } -} diff --git a/weed/worker/tasks/s3_lifecycle/handler.go b/weed/worker/tasks/s3_lifecycle/handler.go index 7e927e9db..9fa0aed1c 100644 --- a/weed/worker/tasks/s3_lifecycle/handler.go +++ b/weed/worker/tasks/s3_lifecycle/handler.go @@ -17,7 +17,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler" - "github.com/seaweedfs/seaweedfs/weed/util" "golang.org/x/time/rate" "google.golang.org/grpc" ) @@ -69,28 +68,17 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { AdminConfigForm: &plugin_pb.ConfigForm{ FormId: "s3-lifecycle-admin", Title: "S3 Lifecycle", - Description: "Cluster-wide controls for the lifecycle scheduler.", + Description: "Cluster-wide delete-throughput cap.", Sections: []*plugin_pb.ConfigSection{ { SectionId: "scope", Title: "Scope", - Description: "Cluster-wide algorithm choice and delete-throughput cap.", + Description: "Cluster-wide delete-throughput cap.", Fields: []*plugin_pb.ConfigField{ - { - Name: "algorithm", - Label: "Algorithm", - Description: "Daily Replay = bounded daily meta-log scan (Phase 2, replay-only — buckets using ExpirationDate, ExpiredDeleteMarker, NewerNoncurrent, or scan_only rules will fail the run until Phase 4 ships). Streaming = legacy reader+heap path, kept as a runtime escape hatch during the Phase 4 rollout; Phase 5 deletes it.", - FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM, - Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_SELECT, - Options: []*plugin_pb.ConfigOption{ - {Value: AlgorithmDailyReplay, Label: "Daily Replay (default)", Description: "Bounded daily meta-log scan. Replay-only in Phase 2; buckets with walker-bound rules fail the run."}, - {Value: AlgorithmStreaming, Label: "Streaming (legacy fallback)", Description: "Long-running reader + per-shard heap + tick dispatcher. Pre-cutover behavior; removed in Phase 5."}, - }, - }, { Name: ClusterDeletesPerSecondAdminKey, Label: "Cluster Delete Rate (per second)", - Description: "Cluster-wide ceiling on lifecycle delete RPCs per second, divided evenly across active s3_lifecycle workers at job-dispatch time. 0 = unlimited (legacy behavior). Only honored by the Daily Replay algorithm; streaming ignores it.", + Description: "Cluster-wide ceiling on lifecycle delete RPCs per second, divided evenly across active s3_lifecycle workers at job-dispatch time. 0 = unlimited.", FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, @@ -107,7 +95,6 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { }, }, DefaultValues: map[string]*plugin_pb.ConfigValue{ - "algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAlgorithm}}, ClusterDeletesPerSecondAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, ClusterDeletesBurstAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, }, @@ -115,49 +102,17 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { WorkerConfigForm: &plugin_pb.ConfigForm{ FormId: "s3-lifecycle-worker", Title: "S3 Lifecycle Worker", - Description: "Operational tuning for the lifecycle pipeline.", + Description: "Operational tuning for the daily lifecycle run.", Sections: []*plugin_pb.ConfigSection{ { SectionId: "cadence", Title: "Cadence", - Description: "How often the worker checks its schedule, saves progress, reloads bucket lifecycle configs, and how long each run may take.", + Description: "Per-run wall-clock budget.", Fields: []*plugin_pb.ConfigField{ - { - Name: "dispatch_tick_minutes", - Label: "Schedule Check Interval (minutes)", - Description: "How often each pipeline drains its schedule.", - FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, - Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, - MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, - }, - { - Name: "checkpoint_tick_seconds", - Label: "Progress Save Interval (seconds)", - Description: "How often each pipeline persists its cursor map to the filer.", - FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, - Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, - MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, - }, - { - Name: "refresh_interval_minutes", - Label: "Lifecycle Config Reload (minutes)", - Description: "How often the scheduler rebuilds the engine snapshot from bucket lifecycle configs.", - FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, - Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, - MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, - }, - { - Name: "bootstrap_interval_minutes", - Label: "Full Bucket Rescan Interval (minutes)", - Description: "How often each bucket is re-walked. scan_only rules — those whose retention horizon exceeds meta-log retention — only fire from the bootstrap walk, so a non-zero value is required to enforce them on a long-running worker. 0 keeps the legacy walk-once-per-process behavior.", - FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, - Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, - MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, - }, { Name: "max_runtime_minutes", Label: "Per-Run Time Limit (minutes)", - Description: "Wall-clock cap on each run. Each daily run processes events for one day; the cursor persists across runs.", + Description: "Wall-clock cap on each run. Each daily run processes one day of meta-log events plus the walker pass over the current rule set; the cursor persists across runs.", FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, @@ -166,11 +121,7 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { }, }, DefaultValues: map[string]*plugin_pb.ConfigValue{ - "dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultDispatchTickMinutes}}, - "checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultCheckpointTickSeconds}}, - "refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultRefreshIntervalMinutes}}, - "bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultBootstrapIntervalMinutes}}, - "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultMaxRuntimeMinutes}}, + "max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultMaxRuntimeMinutes}}, }, }, AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ @@ -272,29 +223,7 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ defer s3Conn.Close() rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn) - if cfg.Algorithm == AlgorithmDailyReplay { - return h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender) - } - - sched := &scheduler.Scheduler{ - BucketsPath: bucketsPath, - Engine: engine.New(), - Persister: &dispatcher.FilerPersister{Store: dispatcher.NewFilerStoreClient(filerClient)}, - Client: lifecycleRPCAdapter{c: rpc}, - FilerClient: filerClient, - ClientID: util.RandomInt32(), - ClientName: "worker-s3-lifecycle", - Workers: cfg.Workers, - DispatchTick: cfg.DispatchTick, - CheckpointTick: cfg.CheckpointTick, - RefreshInterval: cfg.RefreshInterval, - BootstrapInterval: cfg.BootstrapInterval, - } - if err := sched.Run(runCtx); err != nil { - glog.Warningf("s3 lifecycle execute: %v", err) - return err - } - return nil + return h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender) } // executeDailyReplay runs the bounded daily-replay path. Reuses the diff --git a/weed/worker/tasks/s3_lifecycle/handler_test.go b/weed/worker/tasks/s3_lifecycle/handler_test.go index 9b326b636..1ea8331fc 100644 --- a/weed/worker/tasks/s3_lifecycle/handler_test.go +++ b/weed/worker/tasks/s3_lifecycle/handler_test.go @@ -313,11 +313,7 @@ func TestDescriptor_WorkerConfigFormCadenceDefaultsMatchParseConfig(t *testing.T assert.Equal(t, "s3-lifecycle-worker", d.WorkerConfigForm.FormId) wantDefaults := map[string]int64{ - "dispatch_tick_minutes": defaultDispatchTickMinutes, - "checkpoint_tick_seconds": defaultCheckpointTickSeconds, - "refresh_interval_minutes": defaultRefreshIntervalMinutes, - "bootstrap_interval_minutes": defaultBootstrapIntervalMinutes, - "max_runtime_minutes": defaultMaxRuntimeMinutes, + "max_runtime_minutes": defaultMaxRuntimeMinutes, } for name, want := range wantDefaults { t.Run(name, func(t *testing.T) { @@ -326,11 +322,6 @@ func TestDescriptor_WorkerConfigFormCadenceDefaultsMatchParseConfig(t *testing.T assert.Equal(t, want, dv.GetInt64Value(), "default mismatch for %q", name) }) } - // And the form fields themselves: every default must be paired with - // a field of matching name AND INT64 type so the admin can render + - // edit it and ParseConfig's readInt64 reads it correctly. Drift to - // e.g. STRING here would silently make the worker fall back to the - // hardcoded default and ignore admin edits. declared := map[string]plugin_pb.ConfigFieldType{} for _, sec := range d.WorkerConfigForm.Sections { for _, f := range sec.Fields {