mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-02 22:27:14 +00:00
Chris LuandGitHub
e482e67971
The collection filter was parsed twice with two syntaxes: the master-side volume listing compiled the whole string as one regex, while EC encode and EC balance detection split it on commas and matched each entry as a wildcard. A volume had to pass both, so "collection-a,collection-b" matched nothing (no collection is named that), and the ALL_COLLECTIONS sentinel, which the master side skips, dropped every volume at the task side. Parse it once, in one place: a comma-separated list where an entry is a name with optional * and ? wildcards, or a regex when it carries regex syntax. A regex entry now has to match the whole name unless it anchors itself, so listing a collection no longer picks up its longer namesakes.
256 lines
9.4 KiB
Go
256 lines
9.4 KiB
Go
package erasure_coding
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
|
)
|
|
|
|
// Config extends BaseConfig with erasure coding specific settings
|
|
type Config struct {
|
|
base.BaseConfig
|
|
QuietForSeconds int `json:"quiet_for_seconds"`
|
|
FullnessRatio float64 `json:"fullness_ratio"`
|
|
CollectionFilter string `json:"collection_filter"`
|
|
MinSizeMB int `json:"min_size_mb"`
|
|
PreferredTags []string `json:"preferred_tags"`
|
|
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
|
|
}
|
|
|
|
// NewDefaultConfig creates a new default erasure coding configuration
|
|
func NewDefaultConfig() *Config {
|
|
return &Config{
|
|
BaseConfig: base.BaseConfig{
|
|
Enabled: true,
|
|
ScanIntervalSeconds: 60 * 60, // 1 hour
|
|
MaxConcurrent: 1,
|
|
},
|
|
QuietForSeconds: 3600, // 1 hour, matching the shell ec.encode -quietFor default
|
|
FullnessRatio: 0.95, // 95%, matching the shell ec.encode -fullPercent default
|
|
CollectionFilter: "",
|
|
MinSizeMB: 30, // 30MB (more reasonable than 100MB)
|
|
PreferredTags: nil,
|
|
}
|
|
}
|
|
|
|
// GetConfigSpec returns the configuration schema for erasure coding tasks
|
|
func GetConfigSpec() base.ConfigSpec {
|
|
return base.ConfigSpec{
|
|
Fields: []*config.Field{
|
|
{
|
|
Name: "enabled",
|
|
JSONName: "enabled",
|
|
Type: config.FieldTypeBool,
|
|
DefaultValue: true,
|
|
Required: false,
|
|
DisplayName: "Enable Erasure Coding Tasks",
|
|
Description: "Whether erasure coding tasks should be automatically created",
|
|
HelpText: "Toggle this to enable or disable automatic erasure coding task generation",
|
|
InputType: "checkbox",
|
|
CSSClasses: "form-check-input",
|
|
},
|
|
{
|
|
Name: "scan_interval_seconds",
|
|
JSONName: "scan_interval_seconds",
|
|
Type: config.FieldTypeInterval,
|
|
DefaultValue: 60 * 60,
|
|
MinValue: 10 * 60,
|
|
MaxValue: 24 * 60 * 60,
|
|
Required: true,
|
|
DisplayName: "Scan Interval",
|
|
Description: "How often to scan for volumes needing erasure coding",
|
|
HelpText: "The system will check for volumes that need erasure coding at this interval",
|
|
Placeholder: "1",
|
|
Unit: config.UnitHours,
|
|
InputType: "interval",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "max_concurrent",
|
|
JSONName: "max_concurrent",
|
|
Type: config.FieldTypeInt,
|
|
DefaultValue: 1,
|
|
MinValue: 1,
|
|
MaxValue: 5,
|
|
Required: true,
|
|
DisplayName: "Max Concurrent Tasks",
|
|
Description: "Maximum number of erasure coding tasks that can run simultaneously",
|
|
HelpText: "Limits the number of erasure coding operations running at the same time",
|
|
Placeholder: "1 (default)",
|
|
Unit: config.UnitCount,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "quiet_for_seconds",
|
|
JSONName: "quiet_for_seconds",
|
|
Type: config.FieldTypeInterval,
|
|
DefaultValue: 3600,
|
|
MinValue: 1,
|
|
MaxValue: 3600,
|
|
Required: true,
|
|
DisplayName: "Quiet Period",
|
|
Description: "Minimum time volume must be quiet before erasure coding",
|
|
HelpText: "Volume must not be modified for this duration before erasure coding",
|
|
Placeholder: "60",
|
|
Unit: config.UnitMinutes,
|
|
InputType: "interval",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "fullness_ratio",
|
|
JSONName: "fullness_ratio",
|
|
Type: config.FieldTypeFloat,
|
|
DefaultValue: 0.95,
|
|
MinValue: 0.0001,
|
|
MaxValue: 1.0,
|
|
Required: true,
|
|
DisplayName: "Fullness Ratio",
|
|
Description: "Minimum fullness ratio to trigger erasure coding",
|
|
HelpText: "Only volumes with this fullness ratio or higher will be erasure coded",
|
|
Placeholder: "0.80 (80%)",
|
|
Unit: config.UnitNone,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "collection_filter",
|
|
JSONName: "collection_filter",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Collection Filter",
|
|
Description: "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
|
|
HelpText: "Leave empty to process all collections, or list the collections to process",
|
|
Placeholder: "pictures,videos",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "min_size_mb",
|
|
JSONName: "min_size_mb",
|
|
Type: config.FieldTypeInt,
|
|
DefaultValue: 30,
|
|
MinValue: 1,
|
|
MaxValue: 1000,
|
|
Required: true,
|
|
DisplayName: "Minimum Size (MB)",
|
|
Description: "Minimum volume size to consider for erasure coding",
|
|
HelpText: "Only volumes larger than this size will be considered for erasure coding",
|
|
Placeholder: "30",
|
|
Unit: config.UnitNone,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "preferred_tags",
|
|
JSONName: "preferred_tags",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Preferred Disk Tags",
|
|
Description: "Comma-separated disk tags to prioritize for EC shard placement",
|
|
HelpText: "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
|
|
Placeholder: "fast,ssd",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "replica_placement",
|
|
JSONName: "replica_placement",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Replica Placement",
|
|
Description: "EC shard replica placement constraint (e.g. 020)",
|
|
HelpText: "Leave empty to use the master default replication. When set, the 2nd/3rd digits cap EC shards per rack and per node (best-effort during encode: relaxed rather than failing if the cluster can't satisfy them, then enforced by rebalancing). The 1st (data-center) digit is ignored for EC placement",
|
|
Placeholder: "020",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
|
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
|
// Defensive copy of PreferredTags to prevent external mutation
|
|
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
|
|
return &worker_pb.TaskPolicy{
|
|
Enabled: c.Enabled,
|
|
MaxConcurrent: int32(c.MaxConcurrent),
|
|
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
|
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
|
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
|
|
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{
|
|
FullnessRatio: float64(c.FullnessRatio),
|
|
QuietForSeconds: int32(c.QuietForSeconds),
|
|
MinVolumeSizeMb: int32(c.MinSizeMB),
|
|
CollectionFilter: c.CollectionFilter,
|
|
PreferredTags: preferredTagsCopy,
|
|
ReplicaPlacement: c.ReplicaPlacement,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
|
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
|
if policy == nil {
|
|
return fmt.Errorf("policy is nil")
|
|
}
|
|
|
|
// Set general TaskPolicy fields
|
|
c.Enabled = policy.Enabled
|
|
c.MaxConcurrent = int(policy.MaxConcurrent)
|
|
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
|
|
|
// Set erasure coding-specific fields from the task config
|
|
if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
|
|
c.FullnessRatio = float64(ecConfig.FullnessRatio)
|
|
c.QuietForSeconds = int(ecConfig.QuietForSeconds)
|
|
c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
|
|
c.CollectionFilter = ecConfig.CollectionFilter
|
|
c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
|
|
c.ReplicaPlacement = ecConfig.ReplicaPlacement
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
|
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
|
config := NewDefaultConfig()
|
|
|
|
// Try to load from persistence if available
|
|
if persistence, ok := configPersistence.(interface {
|
|
LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error)
|
|
}); ok {
|
|
policy, err := persistence.LoadErasureCodingTaskPolicy()
|
|
switch {
|
|
case err != nil:
|
|
glog.Warningf("Could not read the persisted erasure coding configuration, falling back to defaults: %v", err)
|
|
case policy == nil:
|
|
glog.V(1).Infof("No erasure coding configuration persisted yet, using defaults")
|
|
default:
|
|
if err := config.FromTaskPolicy(policy); err != nil {
|
|
glog.Warningf("Could not apply the persisted erasure coding configuration, falling back to defaults: %v", err)
|
|
} else {
|
|
glog.V(1).Infof("Loaded erasure coding configuration from persistence")
|
|
return config
|
|
}
|
|
}
|
|
} else if configPersistence != nil {
|
|
// A store was handed in but does not expose the accessor, so the persisted
|
|
// settings are silently ignored - always a wiring bug, never a normal state.
|
|
glog.Warningf("%T cannot provide the persisted erasure coding configuration: it has no LoadErasureCodingTaskPolicy() method, "+
|
|
"so the compiled-in defaults are used and any saved erasure coding settings are ignored", configPersistence)
|
|
}
|
|
|
|
glog.V(1).Infof("Using default erasure coding configuration")
|
|
return config
|
|
}
|