mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-17 12:46:59 +00:00
feat(s3/lifecycle): retire algorithm flag, daily_replay is the only path (Phase 5a) (#9465)
feat(s3/lifecycle): remove algorithm flag, daily_replay is the only path (Phase 5a)
With Phase 4b on master the daily_replay path covers every rule kind
and the streaming algorithm has no remaining responsibilities. This
PR retires the algorithm flag from the worker:
- Drop the "Algorithm" enum field from AdminConfigForm and its
DefaultValues entry.
- Drop the if/else routing in Execute — every Execute call now
routes straight into executeDailyReplay.
- Drop the streaming-only worker fields (DispatchTick,
CheckpointTick, RefreshInterval, BootstrapInterval) and their
matching form fields. None of them are read by the daily_replay
path; keeping them in the form would suggest tuning knobs that
don't do anything.
- Drop AlgorithmStreaming / AlgorithmDailyReplay constants and the
Config.Algorithm field.
The streaming-path packages (s3lifecycle/scheduler, s3lifecycle/dispatcher)
remain on the tree; they're now reachable only by the
weed shell s3.lifecycle.run-shard debug command and the few helpers
(LoadCompileInputs, FilerStore, FilerSiblingLister) the daily_replay
worker still uses. Phase 5b deletes the dead code.
Tests prune the cadence-default assertions to the single remaining
field (max_runtime_minutes).
This commit is contained in:
@@ -9,80 +9,25 @@ import (
|
||||
const (
|
||||
jobType = "s3_lifecycle"
|
||||
|
||||
// shardPipelineGoroutines is the in-process fan-out across the
|
||||
// 16-shard space. Kept as a hardcoded internal default — formerly
|
||||
// an admin form field, removed because it's a per-worker tuning
|
||||
// knob, not a cluster-coordination concern.
|
||||
// In-process fan-out across the 16 shards.
|
||||
shardPipelineGoroutines = 1
|
||||
|
||||
defaultDispatchTickMinutes = int64(1)
|
||||
defaultCheckpointTickSeconds = int64(30)
|
||||
defaultRefreshIntervalMinutes = int64(5)
|
||||
defaultMaxRuntimeMinutes = int64(60)
|
||||
defaultBootstrapIntervalMinutes = int64(0) // 0 = walk once per process
|
||||
|
||||
// AlgorithmDailyReplay routes the worker through dailyrun.Run for
|
||||
// one bounded pass per Execute. Currently Phase 2 / replay-only:
|
||||
// buckets with walker-bound action kinds are refused. Phase 4
|
||||
// extends this to handle every kind. Default — the streaming path
|
||||
// stays in the tree as a runtime escape hatch only.
|
||||
AlgorithmDailyReplay = "daily_replay"
|
||||
// AlgorithmStreaming is the legacy event-driven dispatcher path
|
||||
// (reader + heap + per-shard pipeline). Kept as a fallback knob for
|
||||
// rollout; deleted by Phase 5 once Phase 4 walker integration ships.
|
||||
AlgorithmStreaming = "streaming"
|
||||
|
||||
defaultAlgorithm = AlgorithmDailyReplay
|
||||
defaultMaxRuntimeMinutes = int64(60)
|
||||
)
|
||||
|
||||
// Config is the parsed AdminConfigForm + WorkerConfigForm view.
|
||||
type Config struct {
|
||||
Workers int
|
||||
DispatchTick time.Duration
|
||||
CheckpointTick time.Duration
|
||||
RefreshInterval time.Duration
|
||||
BootstrapInterval time.Duration
|
||||
MaxRuntime time.Duration
|
||||
Algorithm string
|
||||
Workers int
|
||||
MaxRuntime time.Duration
|
||||
}
|
||||
|
||||
// ParseConfig pulls the lifecycle Handler config from the merged
|
||||
// admin+worker config values. Missing fields fall back to defaults.
|
||||
func ParseConfig(adminValues, workerValues map[string]*plugin_pb.ConfigValue) Config {
|
||||
func ParseConfig(_ map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config {
|
||||
cfg := Config{
|
||||
Workers: shardPipelineGoroutines,
|
||||
DispatchTick: time.Duration(readInt64(workerValues, "dispatch_tick_minutes", defaultDispatchTickMinutes)) * time.Minute,
|
||||
CheckpointTick: time.Duration(readInt64(workerValues, "checkpoint_tick_seconds", defaultCheckpointTickSeconds)) * time.Second,
|
||||
RefreshInterval: time.Duration(readInt64(workerValues, "refresh_interval_minutes", defaultRefreshIntervalMinutes)) * time.Minute,
|
||||
BootstrapInterval: time.Duration(readInt64(workerValues, "bootstrap_interval_minutes", defaultBootstrapIntervalMinutes)) * time.Minute,
|
||||
MaxRuntime: time.Duration(readInt64(workerValues, "max_runtime_minutes", defaultMaxRuntimeMinutes)) * time.Minute,
|
||||
Algorithm: readString(adminValues, "algorithm", defaultAlgorithm),
|
||||
}
|
||||
if cfg.DispatchTick <= 0 {
|
||||
cfg.DispatchTick = time.Duration(defaultDispatchTickMinutes) * time.Minute
|
||||
}
|
||||
if cfg.CheckpointTick <= 0 {
|
||||
cfg.CheckpointTick = time.Duration(defaultCheckpointTickSeconds) * time.Second
|
||||
}
|
||||
if cfg.RefreshInterval <= 0 {
|
||||
cfg.RefreshInterval = time.Duration(defaultRefreshIntervalMinutes) * time.Minute
|
||||
}
|
||||
// BootstrapInterval is intentionally NOT clamped — zero means
|
||||
// "walk once per process", which is the legacy default for any
|
||||
// deployment that hasn't opted into a cadence yet. Negative values
|
||||
// fall back to zero.
|
||||
if cfg.BootstrapInterval < 0 {
|
||||
cfg.BootstrapInterval = 0
|
||||
Workers: shardPipelineGoroutines,
|
||||
MaxRuntime: time.Duration(readInt64(workerValues, "max_runtime_minutes", defaultMaxRuntimeMinutes)) * time.Minute,
|
||||
}
|
||||
if cfg.MaxRuntime <= 0 {
|
||||
cfg.MaxRuntime = time.Duration(defaultMaxRuntimeMinutes) * time.Minute
|
||||
}
|
||||
switch cfg.Algorithm {
|
||||
case AlgorithmStreaming, AlgorithmDailyReplay:
|
||||
// valid
|
||||
default:
|
||||
cfg.Algorithm = defaultAlgorithm
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -12,112 +12,27 @@ func TestParseConfigDefaults(t *testing.T) {
|
||||
if cfg.Workers != shardPipelineGoroutines {
|
||||
t.Errorf("Workers default=%d, want %d", cfg.Workers, shardPipelineGoroutines)
|
||||
}
|
||||
if cfg.DispatchTick != 1*time.Minute {
|
||||
t.Errorf("DispatchTick default=%v, want 1m", cfg.DispatchTick)
|
||||
}
|
||||
if cfg.CheckpointTick != 30*time.Second {
|
||||
t.Errorf("CheckpointTick default=%v, want 30s", cfg.CheckpointTick)
|
||||
}
|
||||
if cfg.RefreshInterval != 5*time.Minute {
|
||||
t.Errorf("RefreshInterval default=%v, want 5m", cfg.RefreshInterval)
|
||||
}
|
||||
if cfg.MaxRuntime != 60*time.Minute {
|
||||
t.Errorf("MaxRuntime default=%v, want 60m", cfg.MaxRuntime)
|
||||
}
|
||||
if cfg.BootstrapInterval != 0 {
|
||||
t.Errorf("BootstrapInterval default=%v, want 0 (walk-once-per-process)", cfg.BootstrapInterval)
|
||||
}
|
||||
if cfg.Algorithm != AlgorithmDailyReplay {
|
||||
t.Errorf("Algorithm default=%q, want %q", cfg.Algorithm, AlgorithmDailyReplay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_AlgorithmStreamingExplicit(t *testing.T) {
|
||||
// Streaming stays available as a rollout escape hatch until Phase 5
|
||||
// deletes it. Operators must be able to opt back into it explicitly.
|
||||
admin := map[string]*plugin_pb.ConfigValue{
|
||||
"algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: AlgorithmStreaming}},
|
||||
}
|
||||
cfg := ParseConfig(admin, nil)
|
||||
if cfg.Algorithm != AlgorithmStreaming {
|
||||
t.Errorf("Algorithm=%q, want %q", cfg.Algorithm, AlgorithmStreaming)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_AlgorithmUnknownValueFallsBackToDefault(t *testing.T) {
|
||||
// Operators should not be able to silently activate a future
|
||||
// algorithm value by typo. Anything not in the enum falls back to
|
||||
// the default (daily_replay).
|
||||
admin := map[string]*plugin_pb.ConfigValue{
|
||||
"algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "future_algo"}},
|
||||
}
|
||||
cfg := ParseConfig(admin, nil)
|
||||
if cfg.Algorithm != AlgorithmDailyReplay {
|
||||
t.Errorf("unknown algorithm=%q must fall back to %q, got %q", "future_algo", AlgorithmDailyReplay, cfg.Algorithm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfigOverrides(t *testing.T) {
|
||||
func TestParseConfigOverrideMaxRuntime(t *testing.T) {
|
||||
worker := map[string]*plugin_pb.ConfigValue{
|
||||
"dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2}},
|
||||
"checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 15}},
|
||||
"refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10}},
|
||||
"bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}},
|
||||
}
|
||||
cfg := ParseConfig(nil, worker)
|
||||
if cfg.DispatchTick != 2*time.Minute {
|
||||
t.Errorf("DispatchTick=%v, want 2m", cfg.DispatchTick)
|
||||
}
|
||||
if cfg.CheckpointTick != 15*time.Second {
|
||||
t.Errorf("CheckpointTick=%v, want 15s", cfg.CheckpointTick)
|
||||
}
|
||||
if cfg.RefreshInterval != 10*time.Minute {
|
||||
t.Errorf("RefreshInterval=%v, want 10m", cfg.RefreshInterval)
|
||||
}
|
||||
if cfg.BootstrapInterval != 120*time.Minute {
|
||||
t.Errorf("BootstrapInterval=%v, want 120m", cfg.BootstrapInterval)
|
||||
}
|
||||
if cfg.MaxRuntime != 120*time.Minute {
|
||||
t.Errorf("MaxRuntime=%v, want 120m", cfg.MaxRuntime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfigClampsZeroAndNegative(t *testing.T) {
|
||||
func TestParseConfigNegativeMaxRuntimeClampsToDefault(t *testing.T) {
|
||||
worker := map[string]*plugin_pb.ConfigValue{
|
||||
"dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
"checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
"refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
"bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}},
|
||||
}
|
||||
cfg := ParseConfig(nil, worker)
|
||||
if cfg.DispatchTick != 1*time.Minute {
|
||||
t.Errorf("zero DispatchTick should clamp to default, got %v", cfg.DispatchTick)
|
||||
}
|
||||
if cfg.CheckpointTick != 30*time.Second {
|
||||
t.Errorf("zero CheckpointTick should clamp to default, got %v", cfg.CheckpointTick)
|
||||
}
|
||||
if cfg.RefreshInterval != 5*time.Minute {
|
||||
t.Errorf("zero RefreshInterval should clamp to default, got %v", cfg.RefreshInterval)
|
||||
}
|
||||
if cfg.BootstrapInterval != 0 {
|
||||
t.Errorf("negative BootstrapInterval should clamp to 0, got %v", cfg.BootstrapInterval)
|
||||
}
|
||||
if cfg.MaxRuntime != 60*time.Minute {
|
||||
t.Errorf("negative MaxRuntime should clamp to default, got %v", cfg.MaxRuntime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfigBootstrapIntervalZeroIsLegacy(t *testing.T) {
|
||||
// Explicit zero stays zero (walk-once-per-process). Existing
|
||||
// deployments that don't set bootstrap_interval_minutes get the
|
||||
// legacy behavior unchanged.
|
||||
worker := map[string]*plugin_pb.ConfigValue{
|
||||
"bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
}
|
||||
cfg := ParseConfig(nil, worker)
|
||||
if cfg.BootstrapInterval != 0 {
|
||||
t.Errorf("zero BootstrapInterval must stay zero, got %v", cfg.BootstrapInterval)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
@@ -69,28 +68,17 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "s3-lifecycle-admin",
|
||||
Title: "S3 Lifecycle",
|
||||
Description: "Cluster-wide controls for the lifecycle scheduler.",
|
||||
Description: "Cluster-wide delete-throughput cap.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "scope",
|
||||
Title: "Scope",
|
||||
Description: "Cluster-wide algorithm choice and delete-throughput cap.",
|
||||
Description: "Cluster-wide delete-throughput cap.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "algorithm",
|
||||
Label: "Algorithm",
|
||||
Description: "Daily Replay = bounded daily meta-log scan (Phase 2, replay-only — buckets using ExpirationDate, ExpiredDeleteMarker, NewerNoncurrent, or scan_only rules will fail the run until Phase 4 ships). Streaming = legacy reader+heap path, kept as a runtime escape hatch during the Phase 4 rollout; Phase 5 deletes it.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_SELECT,
|
||||
Options: []*plugin_pb.ConfigOption{
|
||||
{Value: AlgorithmDailyReplay, Label: "Daily Replay (default)", Description: "Bounded daily meta-log scan. Replay-only in Phase 2; buckets with walker-bound rules fail the run."},
|
||||
{Value: AlgorithmStreaming, Label: "Streaming (legacy fallback)", Description: "Long-running reader + per-shard heap + tick dispatcher. Pre-cutover behavior; removed in Phase 5."},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: ClusterDeletesPerSecondAdminKey,
|
||||
Label: "Cluster Delete Rate (per second)",
|
||||
Description: "Cluster-wide ceiling on lifecycle delete RPCs per second, divided evenly across active s3_lifecycle workers at job-dispatch time. 0 = unlimited (legacy behavior). Only honored by the Daily Replay algorithm; streaming ignores it.",
|
||||
Description: "Cluster-wide ceiling on lifecycle delete RPCs per second, divided evenly across active s3_lifecycle workers at job-dispatch time. 0 = unlimited.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
@@ -107,7 +95,6 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"algorithm": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAlgorithm}},
|
||||
ClusterDeletesPerSecondAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
ClusterDeletesBurstAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
@@ -115,49 +102,17 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
WorkerConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "s3-lifecycle-worker",
|
||||
Title: "S3 Lifecycle Worker",
|
||||
Description: "Operational tuning for the lifecycle pipeline.",
|
||||
Description: "Operational tuning for the daily lifecycle run.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "cadence",
|
||||
Title: "Cadence",
|
||||
Description: "How often the worker checks its schedule, saves progress, reloads bucket lifecycle configs, and how long each run may take.",
|
||||
Description: "Per-run wall-clock budget.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "dispatch_tick_minutes",
|
||||
Label: "Schedule Check Interval (minutes)",
|
||||
Description: "How often each pipeline drains its schedule.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
|
||||
},
|
||||
{
|
||||
Name: "checkpoint_tick_seconds",
|
||||
Label: "Progress Save Interval (seconds)",
|
||||
Description: "How often each pipeline persists its cursor map to the filer.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
|
||||
},
|
||||
{
|
||||
Name: "refresh_interval_minutes",
|
||||
Label: "Lifecycle Config Reload (minutes)",
|
||||
Description: "How often the scheduler rebuilds the engine snapshot from bucket lifecycle configs.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
|
||||
},
|
||||
{
|
||||
Name: "bootstrap_interval_minutes",
|
||||
Label: "Full Bucket Rescan Interval (minutes)",
|
||||
Description: "How often each bucket is re-walked. scan_only rules — those whose retention horizon exceeds meta-log retention — only fire from the bootstrap walk, so a non-zero value is required to enforce them on a long-running worker. 0 keeps the legacy walk-once-per-process behavior.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
{
|
||||
Name: "max_runtime_minutes",
|
||||
Label: "Per-Run Time Limit (minutes)",
|
||||
Description: "Wall-clock cap on each run. Each daily run processes events for one day; the cursor persists across runs.",
|
||||
Description: "Wall-clock cap on each run. Each daily run processes one day of meta-log events plus the walker pass over the current rule set; the cursor persists across runs.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
|
||||
@@ -166,11 +121,7 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"dispatch_tick_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultDispatchTickMinutes}},
|
||||
"checkpoint_tick_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultCheckpointTickSeconds}},
|
||||
"refresh_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultRefreshIntervalMinutes}},
|
||||
"bootstrap_interval_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultBootstrapIntervalMinutes}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultMaxRuntimeMinutes}},
|
||||
"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultMaxRuntimeMinutes}},
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
@@ -272,29 +223,7 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
|
||||
defer s3Conn.Close()
|
||||
rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn)
|
||||
|
||||
if cfg.Algorithm == AlgorithmDailyReplay {
|
||||
return h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender)
|
||||
}
|
||||
|
||||
sched := &scheduler.Scheduler{
|
||||
BucketsPath: bucketsPath,
|
||||
Engine: engine.New(),
|
||||
Persister: &dispatcher.FilerPersister{Store: dispatcher.NewFilerStoreClient(filerClient)},
|
||||
Client: lifecycleRPCAdapter{c: rpc},
|
||||
FilerClient: filerClient,
|
||||
ClientID: util.RandomInt32(),
|
||||
ClientName: "worker-s3-lifecycle",
|
||||
Workers: cfg.Workers,
|
||||
DispatchTick: cfg.DispatchTick,
|
||||
CheckpointTick: cfg.CheckpointTick,
|
||||
RefreshInterval: cfg.RefreshInterval,
|
||||
BootstrapInterval: cfg.BootstrapInterval,
|
||||
}
|
||||
if err := sched.Run(runCtx); err != nil {
|
||||
glog.Warningf("s3 lifecycle execute: %v", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender)
|
||||
}
|
||||
|
||||
// executeDailyReplay runs the bounded daily-replay path. Reuses the
|
||||
|
||||
@@ -313,11 +313,7 @@ func TestDescriptor_WorkerConfigFormCadenceDefaultsMatchParseConfig(t *testing.T
|
||||
assert.Equal(t, "s3-lifecycle-worker", d.WorkerConfigForm.FormId)
|
||||
|
||||
wantDefaults := map[string]int64{
|
||||
"dispatch_tick_minutes": defaultDispatchTickMinutes,
|
||||
"checkpoint_tick_seconds": defaultCheckpointTickSeconds,
|
||||
"refresh_interval_minutes": defaultRefreshIntervalMinutes,
|
||||
"bootstrap_interval_minutes": defaultBootstrapIntervalMinutes,
|
||||
"max_runtime_minutes": defaultMaxRuntimeMinutes,
|
||||
"max_runtime_minutes": defaultMaxRuntimeMinutes,
|
||||
}
|
||||
for name, want := range wantDefaults {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
@@ -326,11 +322,6 @@ func TestDescriptor_WorkerConfigFormCadenceDefaultsMatchParseConfig(t *testing.T
|
||||
assert.Equal(t, want, dv.GetInt64Value(), "default mismatch for %q", name)
|
||||
})
|
||||
}
|
||||
// And the form fields themselves: every default must be paired with
|
||||
// a field of matching name AND INT64 type so the admin can render +
|
||||
// edit it and ParseConfig's readInt64 reads it correctly. Drift to
|
||||
// e.g. STRING here would silently make the worker fall back to the
|
||||
// hardcoded default and ignore admin edits.
|
||||
declared := map[string]plugin_pb.ConfigFieldType{}
|
||||
for _, sec := range d.WorkerConfigForm.Sections {
|
||||
for _, f := range sec.Fields {
|
||||
|
||||
Reference in New Issue
Block a user