From 37962e2445302755da03519e07b3a193e4fa9e9c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 11 Jun 2026 11:04:52 -0700 Subject: [PATCH] admin: configure maintenance tasks via admin.toml (#9926) * admin: configure maintenance tasks via admin.toml Maintenance task settings could only be edited in the admin UI and live under /conf, so they silently reverted to defaults whenever the data directory was recreated. An optional admin.toml now declares vacuum, balance, and erasure coding settings; keys set there are written through to the persisted task configs at every startup, overriding UI edits, so the configuration stays declarative. Generate an example with "weed scaffold -config=admin". * vacuum: round min volume age up to whole hours MinVolumeAgeSeconds was truncated by integer division when converted to the hour-granular protobuf field, so a sub-hour setting silently became 0 and disabled the age guard. * admin: split and normalize preferred_tags from admin.toml A comma-separated string, as set via environment variable, came through viper as a single slice element. Split on commas and reuse util.NormalizeTagList, matching the plugin config path. * scaffold: clarify admin.toml wording --- weed/admin/dash/config_persistence.go | 2 +- weed/admin/dash/config_toml.go | 127 ++++++++++++++++++ weed/admin/dash/config_toml_test.go | 109 +++++++++++++++ weed/admin/maintenance/maintenance_manager.go | 2 +- weed/command/admin.go | 20 ++- weed/command/scaffold.go | 6 +- weed/command/scaffold/admin.toml | 47 +++++++ weed/command/scaffold/example.go | 3 + weed/worker/tasks/vacuum/config.go | 2 +- 9 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 weed/admin/dash/config_toml.go create mode 100644 weed/admin/dash/config_toml_test.go create mode 100644 weed/command/scaffold/admin.toml diff --git a/weed/admin/dash/config_persistence.go b/weed/admin/dash/config_persistence.go index ef8363835..63d6bcdcc 100644 --- a/weed/admin/dash/config_persistence.go +++ b/weed/admin/dash/config_persistence.go @@ -706,7 +706,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy { TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{ VacuumConfig: &worker_pb.VacuumTaskConfig{ GarbageThreshold: float64(vacuumConfig.GarbageThreshold), - MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours + MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0 }, }, } diff --git a/weed/admin/dash/config_toml.go b/weed/admin/dash/config_toml.go new file mode 100644 index 000000000..81e31da72 --- /dev/null +++ b/weed/admin/dash/config_toml.go @@ -0,0 +1,127 @@ +package dash + +import ( + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" +) + +// TomlConfig is the subset of viper used to read admin.toml values. +type TomlConfig interface { + IsSet(key string) bool + GetBool(key string) bool + GetInt(key string) int + GetFloat64(key string) float64 + GetString(key string) string + GetStringSlice(key string) []string +} + +// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in +// admin.toml through to the persisted task configs, so they survive data +// directory loss and override admin UI edits on restart. Absent keys keep +// their persisted values. +func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error { + vacuumConf := vacuum.LoadConfigFromPersistence(cp) + vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig) + if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) { + vacuumConf.GarbageThreshold = v.GetFloat64(k) + vacuumChanged = true + } + if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) { + vacuumConf.MinVolumeAgeSeconds = v.GetInt(k) + vacuumChanged = true + } + + balanceConf := balance.LoadConfigFromPersistence(cp) + balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig) + if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) { + balanceConf.ImbalanceThreshold = v.GetFloat64(k) + balanceChanged = true + } + if k := "maintenance.balance.min_server_count"; v.IsSet(k) { + balanceConf.MinServerCount = v.GetInt(k) + balanceChanged = true + } + + ecConf := erasure_coding.LoadConfigFromPersistence(cp) + ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig) + if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) { + ecConf.FullnessRatio = v.GetFloat64(k) + ecChanged = true + } + if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) { + ecConf.QuietForSeconds = v.GetInt(k) + ecChanged = true + } + if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) { + ecConf.MinSizeMB = v.GetInt(k) + ecChanged = true + } + if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) { + ecConf.CollectionFilter = v.GetString(k) + ecChanged = true + } + if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) { + // viper does not split comma-separated values from env vars + var tags []string + for _, tag := range v.GetStringSlice(k) { + tags = append(tags, strings.Split(tag, ",")...) + } + ecConf.PreferredTags = util.NormalizeTagList(tags) + ecChanged = true + } + if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) { + ecConf.ReplicaPlacement = v.GetString(k) + ecChanged = true + } + + if !vacuumChanged && !balanceChanged && !ecChanged { + return nil + } + if !cp.IsConfigured() { + return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist") + } + + if vacuumChanged { + if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil { + return fmt.Errorf("save vacuum task config: %w", err) + } + glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml") + } + if balanceChanged { + if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil { + return fmt.Errorf("save balance task config: %w", err) + } + glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml") + } + if ecChanged { + if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil { + return fmt.Errorf("save erasure coding task config: %w", err) + } + glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml") + } + return nil +} + +func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool { + changed := false + if k := prefix + "enabled"; v.IsSet(k) { + c.Enabled = v.GetBool(k) + changed = true + } + if k := prefix + "scan_interval_seconds"; v.IsSet(k) { + c.ScanIntervalSeconds = v.GetInt(k) + changed = true + } + if k := prefix + "max_concurrent"; v.IsSet(k) { + c.MaxConcurrent = v.GetInt(k) + changed = true + } + return changed +} diff --git a/weed/admin/dash/config_toml_test.go b/weed/admin/dash/config_toml_test.go new file mode 100644 index 000000000..4e55a8119 --- /dev/null +++ b/weed/admin/dash/config_toml_test.go @@ -0,0 +1,109 @@ +package dash + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" + "github.com/spf13/viper" +) + +func tomlConfig(t *testing.T, content string) *viper.Viper { + t.Helper() + v := viper.New() + v.SetConfigType("toml") + if err := v.ReadConfig(strings.NewReader(content)); err != nil { + t.Fatalf("read toml: %v", err) + } + return v +} + +func TestApplyMaintenanceConfigFromToml(t *testing.T) { + dir := t.TempDir() + cp := NewConfigPersistence(dir) + + // simulate an earlier UI edit + saved := vacuum.NewDefaultConfig() + saved.MaxConcurrent = 5 + saved.GarbageThreshold = 0.5 + if err := cp.SaveVacuumTaskPolicy(saved.ToTaskPolicy()); err != nil { + t.Fatalf("save vacuum policy: %v", err) + } + + balanceBefore := balance.LoadConfigFromPersistence(cp) + + v := tomlConfig(t, ` +[maintenance.vacuum] +garbage_threshold = 0.03 +min_volume_age_seconds = 1800 + +[maintenance.erasure_coding] +enabled = false +fullness_ratio = 0.8 +preferred_tags = "Fast, ssd" +`) + if err := cp.ApplyMaintenanceConfigFromToml(v); err != nil { + t.Fatalf("apply: %v", err) + } + + vacuumConf := vacuum.LoadConfigFromPersistence(cp) + if vacuumConf.GarbageThreshold != 0.03 { + t.Errorf("garbage threshold = %v, want 0.03", vacuumConf.GarbageThreshold) + } + // sub-hour values round up to a whole hour instead of truncating to 0 + if vacuumConf.MinVolumeAgeSeconds != 3600 { + t.Errorf("min volume age = %v, want 3600", vacuumConf.MinVolumeAgeSeconds) + } + if vacuumConf.MaxConcurrent != 5 { + t.Errorf("max concurrent = %v, want UI-saved 5 preserved", vacuumConf.MaxConcurrent) + } + + ecConf := erasure_coding.LoadConfigFromPersistence(cp) + if ecConf.Enabled { + t.Errorf("ec enabled = true, want false") + } + if ecConf.FullnessRatio != 0.8 { + t.Errorf("fullness ratio = %v, want 0.8", ecConf.FullnessRatio) + } + if !reflect.DeepEqual(ecConf.PreferredTags, []string{"fast", "ssd"}) { + t.Errorf("preferred tags = %v, want [fast ssd]", ecConf.PreferredTags) + } + if ecConf.QuietForSeconds != 3600 { + t.Errorf("quiet for = %v, want default 3600 preserved", ecConf.QuietForSeconds) + } + + // balance section absent: nothing written + if _, err := os.Stat(filepath.Join(dir, ConfigSubdir, BalanceTaskConfigFile)); !os.IsNotExist(err) { + t.Errorf("balance config written without [maintenance.balance] section") + } + if got := balance.LoadConfigFromPersistence(cp); !reflect.DeepEqual(got, balanceBefore) { + t.Errorf("balance config changed without [maintenance.balance] section") + } +} + +func TestApplyMaintenanceConfigFromTomlNoKeys(t *testing.T) { + dir := t.TempDir() + cp := NewConfigPersistence(dir) + if err := cp.ApplyMaintenanceConfigFromToml(viper.New()); err != nil { + t.Fatalf("apply: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, ConfigSubdir)); !os.IsNotExist(err) { + t.Errorf("conf dir created without any maintenance keys set") + } +} + +func TestApplyMaintenanceConfigFromTomlRequiresDataDir(t *testing.T) { + cp := NewConfigPersistence("") + v := tomlConfig(t, ` +[maintenance.vacuum] +garbage_threshold = 0.03 +`) + if err := cp.ApplyMaintenanceConfigFromToml(v); err == nil { + t.Errorf("expected error when maintenance keys are set without a data dir") + } +} diff --git a/weed/admin/maintenance/maintenance_manager.go b/weed/admin/maintenance/maintenance_manager.go index 9aaf2ebfa..509cdc899 100644 --- a/weed/admin/maintenance/maintenance_manager.go +++ b/weed/admin/maintenance/maintenance_manager.go @@ -33,7 +33,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy { TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{ VacuumConfig: &worker_pb.VacuumTaskConfig{ GarbageThreshold: float64(vacuumConfig.GarbageThreshold), - MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours + MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0 }, }, } diff --git a/weed/command/admin.go b/weed/command/admin.go index c3a699082..daa1b5cdc 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -171,8 +171,17 @@ var cmdAdmin = &Command{ - Metrics are disabled when -metricsPort is 0 (the default) - Example: weed admin -metricsPort=9327 -master="localhost:9333" + Maintenance Configuration: + - An optional admin.toml declares maintenance task settings + ([maintenance.vacuum], [maintenance.balance], [maintenance.erasure_coding]) + - Settings in admin.toml are applied at every startup, overriding values + saved from the admin UI, so they can be managed declaratively + - Requires -dataDir; values can also be set via WEED_* environment + variables, e.g. WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3 + - Generate example admin.toml: weed scaffold -config=admin + Configuration File: - - The security.toml file is read from ".", "$HOME/.seaweedfs/", + - The security.toml and admin.toml files are read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order - Generate example security.toml: weed scaffold -config=security @@ -191,6 +200,9 @@ func runAdmin(cmd *Command, args []string) bool { // Load security configuration util.LoadSecurityConfiguration() + // Optional admin.toml with maintenance task settings + util.LoadConfiguration("admin", false) + // Apply security.toml / env var fallbacks for credential flags. // CLI flags take precedence over security.toml / WEED_* env vars. applyViperFallback(cmd, a.adminUser, "adminUser", "admin.user") @@ -337,6 +349,12 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, glog.Infof("Data directory created/verified: %s", dataDir) } + // Write maintenance task settings from admin.toml into the persisted + // task configs before the server loads them + if err := dash.NewConfigPersistence(dataDir).ApplyMaintenanceConfigFromToml(util.GetViper()); err != nil { + return fmt.Errorf("apply admin.toml: %w", err) + } + // Detect TLS configuration to set Secure cookie flag cookieSecure := viper.GetString("https.admin.key") != "" diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 3530bab84..ea087ed4e 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -14,7 +14,7 @@ func init() { } var cmdScaffold = &Command{ - UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential]", + UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential|admin]", Short: "generate basic configuration files", Long: `Generate configuration files with all possible configurations for you to customize. @@ -31,7 +31,7 @@ var cmdScaffold = &Command{ var ( outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory") - config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential] the configuration file to generate") + config = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential|admin] the configuration file to generate") ) func runScaffold(cmd *Command, args []string) bool { @@ -54,6 +54,8 @@ func runScaffold(cmd *Command, args []string) bool { content = scaffold.Shell case "credential": content = scaffold.Credential + case "admin": + content = scaffold.Admin } if content == "" { println("need a valid -config option") diff --git a/weed/command/scaffold/admin.toml b/weed/command/scaffold/admin.toml new file mode 100644 index 000000000..679053e96 --- /dev/null +++ b/weed/command/scaffold/admin.toml @@ -0,0 +1,47 @@ +# Put this file in one of the locations, with descending priority +# ./admin.toml +# $HOME/.seaweedfs/admin.toml +# /etc/seaweedfs/admin.toml +# this file is read by "weed admin" + +# Maintenance task settings declared here are applied every time the admin +# server starts, overriding values saved from the admin UI under +# /conf. Commented-out keys keep their saved or default values. +# Requires -dataDir. +# Each value can also be set via environment variable, e.g. +# export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3 + +[maintenance.vacuum] +# enabled = true +# vacuum volumes with more deleted content than this ratio +# garbage_threshold = 0.3 +# how often to scan for volumes needing vacuum +# scan_interval_seconds = 7200 +# max_concurrent = 2 +# only vacuum volumes older than this (rounded up to whole hours) +# min_volume_age_seconds = 86400 + +[maintenance.balance] +# enabled = true +# trigger balancing when volume distribution exceeds this ratio +# imbalance_threshold = 0.2 +# scan_interval_seconds = 1800 +# max_concurrent = 1 +# minimum number of volume servers before balancing +# min_server_count = 2 + +[maintenance.erasure_coding] +# enabled = true +# only erasure code volumes this full +# fullness_ratio = 0.95 +# only erasure code volumes not modified for this long +# quiet_for_seconds = 3600 +# scan_interval_seconds = 3600 +# max_concurrent = 1 +# min_size_mb = 30 +# only process volumes from this collection ("" = all collections) +# collection_filter = "" +# disk tags preferred for shard placement +# preferred_tags = ["fast", "ssd"] +# EC shard placement constraint, e.g. "020"; empty uses the master default replication +# replica_placement = "" diff --git a/weed/command/scaffold/example.go b/weed/command/scaffold/example.go index 55259fae8..de645ac8c 100644 --- a/weed/command/scaffold/example.go +++ b/weed/command/scaffold/example.go @@ -35,3 +35,6 @@ var Shell string //go:embed credential.toml var Credential string + +//go:embed admin.toml +var Admin string diff --git a/weed/worker/tasks/vacuum/config.go b/weed/worker/tasks/vacuum/config.go index a2a45d57d..5d4469707 100644 --- a/weed/worker/tasks/vacuum/config.go +++ b/weed/worker/tasks/vacuum/config.go @@ -39,7 +39,7 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy { TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{ VacuumConfig: &worker_pb.VacuumTaskConfig{ GarbageThreshold: float64(c.GarbageThreshold), - MinVolumeAgeHours: int32(c.MinVolumeAgeSeconds / 3600), // Convert seconds to hours + MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0 }, }, }