admin: configure maintenance tasks via admin.toml (#9926)

* admin: configure maintenance tasks via admin.toml

Maintenance task settings could only be edited in the admin UI and live
under <dataDir>/conf, so they silently reverted to defaults whenever the
data directory was recreated. An optional admin.toml now declares vacuum,
balance, and erasure coding settings; keys set there are written through
to the persisted task configs at every startup, overriding UI edits, so
the configuration stays declarative. Generate an example with
"weed scaffold -config=admin".

* vacuum: round min volume age up to whole hours

MinVolumeAgeSeconds was truncated by integer division when converted to
the hour-granular protobuf field, so a sub-hour setting silently became
0 and disabled the age guard.

* admin: split and normalize preferred_tags from admin.toml

A comma-separated string, as set via environment variable, came through
viper as a single slice element. Split on commas and reuse
util.NormalizeTagList, matching the plugin config path.

* scaffold: clarify admin.toml wording

weed/admin/dash/config_persistence.go

@@ -706,7 +706,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {
 			TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 				VacuumConfig: &worker_pb.VacuumTaskConfig{
 					GarbageThreshold:  float64(vacuumConfig.GarbageThreshold),
-					MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+					MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
 				},
 			},
 		}

weed/admin/dash/config_toml.go

@@ -0,0 +1,127 @@
+package dash
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+)
+
+// TomlConfig is the subset of viper used to read admin.toml values.
+type TomlConfig interface {
+	IsSet(key string) bool
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetFloat64(key string) float64
+	GetString(key string) string
+	GetStringSlice(key string) []string
+}
+
+// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in
+// admin.toml through to the persisted task configs, so they survive data
+// directory loss and override admin UI edits on restart. Absent keys keep
+// their persisted values.
+func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error {
+	vacuumConf := vacuum.LoadConfigFromPersistence(cp)
+	vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig)
+	if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) {
+		vacuumConf.GarbageThreshold = v.GetFloat64(k)
+		vacuumChanged = true
+	}
+	if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) {
+		vacuumConf.MinVolumeAgeSeconds = v.GetInt(k)
+		vacuumChanged = true
+	}
+
+	balanceConf := balance.LoadConfigFromPersistence(cp)
+	balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig)
+	if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) {
+		balanceConf.ImbalanceThreshold = v.GetFloat64(k)
+		balanceChanged = true
+	}
+	if k := "maintenance.balance.min_server_count"; v.IsSet(k) {
+		balanceConf.MinServerCount = v.GetInt(k)
+		balanceChanged = true
+	}
+
+	ecConf := erasure_coding.LoadConfigFromPersistence(cp)
+	ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig)
+	if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) {
+		ecConf.FullnessRatio = v.GetFloat64(k)
+		ecChanged = true
+	}
+	if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) {
+		ecConf.QuietForSeconds = v.GetInt(k)
+		ecChanged = true
+	}
+	if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) {
+		ecConf.MinSizeMB = v.GetInt(k)
+		ecChanged = true
+	}
+	if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) {
+		ecConf.CollectionFilter = v.GetString(k)
+		ecChanged = true
+	}
+	if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) {
+		// viper does not split comma-separated values from env vars
+		var tags []string
+		for _, tag := range v.GetStringSlice(k) {
+			tags = append(tags, strings.Split(tag, ",")...)
+		}
+		ecConf.PreferredTags = util.NormalizeTagList(tags)
+		ecChanged = true
+	}
+	if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) {
+		ecConf.ReplicaPlacement = v.GetString(k)
+		ecChanged = true
+	}
+
+	if !vacuumChanged && !balanceChanged && !ecChanged {
+		return nil
+	}
+	if !cp.IsConfigured() {
+		return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist")
+	}
+
+	if vacuumChanged {
+		if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil {
+			return fmt.Errorf("save vacuum task config: %w", err)
+		}
+		glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml")
+	}
+	if balanceChanged {
+		if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil {
+			return fmt.Errorf("save balance task config: %w", err)
+		}
+		glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml")
+	}
+	if ecChanged {
+		if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil {
+			return fmt.Errorf("save erasure coding task config: %w", err)
+		}
+		glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml")
+	}
+	return nil
+}
+
+func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool {
+	changed := false
+	if k := prefix + "enabled"; v.IsSet(k) {
+		c.Enabled = v.GetBool(k)
+		changed = true
+	}
+	if k := prefix + "scan_interval_seconds"; v.IsSet(k) {
+		c.ScanIntervalSeconds = v.GetInt(k)
+		changed = true
+	}
+	if k := prefix + "max_concurrent"; v.IsSet(k) {
+		c.MaxConcurrent = v.GetInt(k)
+		changed = true
+	}
+	return changed
+}

weed/admin/dash/config_toml_test.go

@@ -0,0 +1,109 @@
+package dash
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	"github.com/spf13/viper"
+)
+
+func tomlConfig(t *testing.T, content string) *viper.Viper {
+	t.Helper()
+	v := viper.New()
+	v.SetConfigType("toml")
+	if err := v.ReadConfig(strings.NewReader(content)); err != nil {
+		t.Fatalf("read toml: %v", err)
+	}
+	return v
+}
+
+func TestApplyMaintenanceConfigFromToml(t *testing.T) {
+	dir := t.TempDir()
+	cp := NewConfigPersistence(dir)
+
+	// simulate an earlier UI edit
+	saved := vacuum.NewDefaultConfig()
+	saved.MaxConcurrent = 5
+	saved.GarbageThreshold = 0.5
+	if err := cp.SaveVacuumTaskPolicy(saved.ToTaskPolicy()); err != nil {
+		t.Fatalf("save vacuum policy: %v", err)
+	}
+
+	balanceBefore := balance.LoadConfigFromPersistence(cp)
+
+	v := tomlConfig(t, `
+[maintenance.vacuum]
+garbage_threshold = 0.03
+min_volume_age_seconds = 1800
+
+[maintenance.erasure_coding]
+enabled = false
+fullness_ratio = 0.8
+preferred_tags = "Fast, ssd"
+`)
+	if err := cp.ApplyMaintenanceConfigFromToml(v); err != nil {
+		t.Fatalf("apply: %v", err)
+	}
+
+	vacuumConf := vacuum.LoadConfigFromPersistence(cp)
+	if vacuumConf.GarbageThreshold != 0.03 {
+		t.Errorf("garbage threshold = %v, want 0.03", vacuumConf.GarbageThreshold)
+	}
+	// sub-hour values round up to a whole hour instead of truncating to 0
+	if vacuumConf.MinVolumeAgeSeconds != 3600 {
+		t.Errorf("min volume age = %v, want 3600", vacuumConf.MinVolumeAgeSeconds)
+	}
+	if vacuumConf.MaxConcurrent != 5 {
+		t.Errorf("max concurrent = %v, want UI-saved 5 preserved", vacuumConf.MaxConcurrent)
+	}
+
+	ecConf := erasure_coding.LoadConfigFromPersistence(cp)
+	if ecConf.Enabled {
+		t.Errorf("ec enabled = true, want false")
+	}
+	if ecConf.FullnessRatio != 0.8 {
+		t.Errorf("fullness ratio = %v, want 0.8", ecConf.FullnessRatio)
+	}
+	if !reflect.DeepEqual(ecConf.PreferredTags, []string{"fast", "ssd"}) {
+		t.Errorf("preferred tags = %v, want [fast ssd]", ecConf.PreferredTags)
+	}
+	if ecConf.QuietForSeconds != 3600 {
+		t.Errorf("quiet for = %v, want default 3600 preserved", ecConf.QuietForSeconds)
+	}
+
+	// balance section absent: nothing written
+	if _, err := os.Stat(filepath.Join(dir, ConfigSubdir, BalanceTaskConfigFile)); !os.IsNotExist(err) {
+		t.Errorf("balance config written without [maintenance.balance] section")
+	}
+	if got := balance.LoadConfigFromPersistence(cp); !reflect.DeepEqual(got, balanceBefore) {
+		t.Errorf("balance config changed without [maintenance.balance] section")
+	}
+}
+
+func TestApplyMaintenanceConfigFromTomlNoKeys(t *testing.T) {
+	dir := t.TempDir()
+	cp := NewConfigPersistence(dir)
+	if err := cp.ApplyMaintenanceConfigFromToml(viper.New()); err != nil {
+		t.Fatalf("apply: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, ConfigSubdir)); !os.IsNotExist(err) {
+		t.Errorf("conf dir created without any maintenance keys set")
+	}
+}
+
+func TestApplyMaintenanceConfigFromTomlRequiresDataDir(t *testing.T) {
+	cp := NewConfigPersistence("")
+	v := tomlConfig(t, `
+[maintenance.vacuum]
+garbage_threshold = 0.03
+`)
+	if err := cp.ApplyMaintenanceConfigFromToml(v); err == nil {
+		t.Errorf("expected error when maintenance keys are set without a data dir")
+	}
+}

weed/admin/maintenance/maintenance_manager.go

@@ -33,7 +33,7 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {
 			TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 				VacuumConfig: &worker_pb.VacuumTaskConfig{
 					GarbageThreshold:  float64(vacuumConfig.GarbageThreshold),
-					MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+					MinVolumeAgeHours: int32((vacuumConfig.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
 				},
 			},
 		}

weed/command/admin.go

@@ -171,8 +171,17 @@ var cmdAdmin = &Command{
     - Metrics are disabled when -metricsPort is 0 (the default)
     - Example: weed admin -metricsPort=9327 -master="localhost:9333"
 
+  Maintenance Configuration:
+    - An optional admin.toml declares maintenance task settings
+      ([maintenance.vacuum], [maintenance.balance], [maintenance.erasure_coding])
+    - Settings in admin.toml are applied at every startup, overriding values
+      saved from the admin UI, so they can be managed declaratively
+    - Requires -dataDir; values can also be set via WEED_* environment
+      variables, e.g. WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
+    - Generate example admin.toml: weed scaffold -config=admin
+
   Configuration File:
-    - The security.toml file is read from ".", "$HOME/.seaweedfs/",
+    - The security.toml and admin.toml files are read from ".", "$HOME/.seaweedfs/",
       "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
     - Generate example security.toml: weed scaffold -config=security
 
@@ -191,6 +200,9 @@ func runAdmin(cmd *Command, args []string) bool {
 	// Load security configuration
 	util.LoadSecurityConfiguration()
 
+	// Optional admin.toml with maintenance task settings
+	util.LoadConfiguration("admin", false)
+
 	// Apply security.toml / env var fallbacks for credential flags.
 	// CLI flags take precedence over security.toml / WEED_* env vars.
 	applyViperFallback(cmd, a.adminUser, "adminUser", "admin.user")
@@ -337,6 +349,12 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 		glog.Infof("Data directory created/verified: %s", dataDir)
 	}
 
+	// Write maintenance task settings from admin.toml into the persisted
+	// task configs before the server loads them
+	if err := dash.NewConfigPersistence(dataDir).ApplyMaintenanceConfigFromToml(util.GetViper()); err != nil {
+		return fmt.Errorf("apply admin.toml: %w", err)
+	}
+
 	// Detect TLS configuration to set Secure cookie flag
 	cookieSecure := viper.GetString("https.admin.key") != ""
 

weed/command/scaffold.go

@@ -14,7 +14,7 @@ func init() {
 }
 
 var cmdScaffold = &Command{
-	UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential]",
+	UsageLine: "scaffold -config=[filer|notification|replication|security|master|volume|shell|credential|admin]",
 	Short:     "generate basic configuration files",
 	Long: `Generate configuration files with all possible configurations for you to customize.
 
@@ -31,7 +31,7 @@ var cmdScaffold = &Command{
 
 var (
 	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
-	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential] the configuration file to generate")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master|volume|shell|credential|admin] the configuration file to generate")
 )
 
 func runScaffold(cmd *Command, args []string) bool {
@@ -54,6 +54,8 @@ func runScaffold(cmd *Command, args []string) bool {
 		content = scaffold.Shell
 	case "credential":
 		content = scaffold.Credential
+	case "admin":
+		content = scaffold.Admin
 	}
 	if content == "" {
 		println("need a valid -config option")

weed/command/scaffold/admin.toml

@@ -0,0 +1,47 @@
+# Put this file in one of the locations, with descending priority
+#    ./admin.toml
+#    $HOME/.seaweedfs/admin.toml
+#    /etc/seaweedfs/admin.toml
+# this file is read by "weed admin"
+
+# Maintenance task settings declared here are applied every time the admin
+# server starts, overriding values saved from the admin UI under
+# <dataDir>/conf. Commented-out keys keep their saved or default values.
+# Requires -dataDir.
+# Each value can also be set via environment variable, e.g.
+#    export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
+
+[maintenance.vacuum]
+# enabled = true
+# vacuum volumes with more deleted content than this ratio
+# garbage_threshold = 0.3
+# how often to scan for volumes needing vacuum
+# scan_interval_seconds = 7200
+# max_concurrent = 2
+# only vacuum volumes older than this (rounded up to whole hours)
+# min_volume_age_seconds = 86400
+
+[maintenance.balance]
+# enabled = true
+# trigger balancing when volume distribution exceeds this ratio
+# imbalance_threshold = 0.2
+# scan_interval_seconds = 1800
+# max_concurrent = 1
+# minimum number of volume servers before balancing
+# min_server_count = 2
+
+[maintenance.erasure_coding]
+# enabled = true
+# only erasure code volumes this full
+# fullness_ratio = 0.95
+# only erasure code volumes not modified for this long
+# quiet_for_seconds = 3600
+# scan_interval_seconds = 3600
+# max_concurrent = 1
+# min_size_mb = 30
+# only process volumes from this collection ("" = all collections)
+# collection_filter = ""
+# disk tags preferred for shard placement
+# preferred_tags = ["fast", "ssd"]
+# EC shard placement constraint, e.g. "020"; empty uses the master default replication
+# replica_placement = ""

weed/command/scaffold/example.go

@@ -35,3 +35,6 @@ var Shell string
 
 //go:embed credential.toml
 var Credential string
+
+//go:embed admin.toml
+var Admin string

weed/worker/tasks/vacuum/config.go

@@ -39,7 +39,7 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
 		TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 			VacuumConfig: &worker_pb.VacuumTaskConfig{
 				GarbageThreshold:  float64(c.GarbageThreshold),
-				MinVolumeAgeHours: int32(c.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+				MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
 			},
 		},
 	}