diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index f360c7653..f01b86bb4 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -132,7 +132,49 @@ jobs: done echo "✅ S3 server is responding, starting tests..." - + + # Spawn the lifecycle worker so test_lifecycle_expiration etc. have + # something driving deletions. The s3tests build tag rescales one + # day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of + # the upload's mtime; -dispatch / -checkpoint defaults are already + # tightened under the same build tag. + LC_LOG=/tmp/lifecycle-worker.log + # -debug routes glog to stderr so the bootstrap walker's progress + # shows up in $LC_LOG; without it weed shell silences glog. + (echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \ + | weed shell -debug -master=localhost:9333 \ + > "$LC_LOG" 2>&1 & + lc_pid=$! + # Aliveness check: a bad shell command exits in <1s and the suite + # would otherwise just timeout the expiration tests with no signal. + sleep 2 + if ! kill -0 "$lc_pid" 2>/dev/null; then + echo "lifecycle worker died on startup" + tail -50 "$LC_LOG" 2>/dev/null || true + exit 1 + fi + echo "lifecycle worker pid=$lc_pid" + + # bash -e exits the step on the first tox failure, so move teardown + # into a trap to guarantee the worker log + data dir reach the runner. + cleanup() { + status=$? + # SIGTERM first so the worker's stdout flushes; SIGKILL is the + # bash fallback if it ignores TERM. Reading the log AFTER the + # graceful-stop window catches the bootstrap walker's progress. + kill -TERM "$lc_pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + if [ "$status" -ne 0 ]; then + echo "=== lifecycle worker log (tail) ===" + tail -200 "$LC_LOG" 2>/dev/null || true + fi + kill -9 "$lc_pid" 2>/dev/null || true + kill -9 "$pid" 2>/dev/null || true + rm -rf "$WEED_DATA_DIR" 2>/dev/null || true + } + trap cleanup EXIT + tox -- \ s3tests/functional/test_s3.py::test_bucket_list_empty \ s3tests/functional/test_s3.py::test_bucket_list_distinct \ @@ -312,11 +354,8 @@ jobs: s3tests/functional/test_s3.py::test_lifecycle_get \ s3tests/functional/test_s3.py::test_lifecycle_set_filter \ s3tests/functional/test_s3.py::test_lifecycle_expiration \ - s3tests/functional/test_s3.py::test_lifecyclev2_expiration \ - s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled - kill -9 $pid || true - # Clean up data directory - rm -rf "$WEED_DATA_DIR" || true + s3tests/functional/test_s3.py::test_lifecyclev2_expiration + # cleanup() trap handles worker/server kill + data dir wipe. versioning-tests: name: S3 Versioning & Object Lock tests @@ -958,6 +997,39 @@ jobs: sleep 2 done + + # Spawn the lifecycle worker (see basic-tests block for context). + LC_LOG=/tmp/lifecycle-worker-sql.log + (echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \ + | weed shell -debug -master=localhost:9337 \ + > "$LC_LOG" 2>&1 & + lc_pid=$! + sleep 2 + if ! kill -0 "$lc_pid" 2>/dev/null; then + echo "lifecycle worker died on startup" + tail -50 "$LC_LOG" 2>/dev/null || true + exit 1 + fi + echo "lifecycle worker pid=$lc_pid" + + cleanup() { + status=$? + # SIGTERM first so the worker's stdout flushes; SIGKILL is the + # bash fallback if it ignores TERM. Reading the log AFTER the + # graceful-stop window catches the bootstrap walker's progress. + kill -TERM "$lc_pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + if [ "$status" -ne 0 ]; then + echo "=== lifecycle worker log (tail) ===" + tail -200 "$LC_LOG" 2>/dev/null || true + fi + kill -9 "$lc_pid" 2>/dev/null || true + kill -9 "$pid" 2>/dev/null || true + rm -rf "$WEED_DATA_DIR" 2>/dev/null || true + } + trap cleanup EXIT + tox -- \ s3tests/functional/test_s3.py::test_bucket_list_empty \ s3tests/functional/test_s3.py::test_bucket_list_distinct \ @@ -1137,10 +1209,7 @@ jobs: s3tests/functional/test_s3.py::test_lifecycle_get \ s3tests/functional/test_s3.py::test_lifecycle_set_filter \ s3tests/functional/test_s3.py::test_lifecycle_expiration \ - s3tests/functional/test_s3.py::test_lifecyclev2_expiration \ - s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled - kill -9 $pid || true - # Clean up data directory - rm -rf "$WEED_DATA_DIR" || true + s3tests/functional/test_s3.py::test_lifecyclev2_expiration + # cleanup() trap handles worker/server kill + data dir wipe. diff --git a/weed/command/worker_test.go b/weed/command/worker_test.go index 6d96af7d4..b2044b9de 100644 --- a/weed/command/worker_test.go +++ b/weed/command/worker_test.go @@ -1,6 +1,7 @@ package command import ( + "sort" "testing" "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" @@ -14,8 +15,26 @@ func TestWorkerDefaultJobTypes(t *testing.T) { if err != nil { t.Fatalf("buildPluginWorkerHandlers(default worker flag) err = %v", err) } - // Expected: vacuum, volume_balance, admin_script, erasure_coding, iceberg_maintenance, ec_balance - if len(handlers) != 6 { - t.Fatalf("expected default worker job types to include 6 handlers, got %d", len(handlers)) + want := []string{ + "admin_script", + "ec_balance", + "erasure_coding", + "iceberg_maintenance", + "s3_lifecycle", + "vacuum", + "volume_balance", + } + got := make([]string, 0, len(handlers)) + for _, h := range handlers { + got = append(got, h.Capability().JobType) + } + sort.Strings(got) + if len(got) != len(want) { + t.Fatalf("default worker job types: got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("default worker job types: got %v, want %v", got, want) + } } } diff --git a/weed/s3api/s3lifecycle/bootstrap/walker_test.go b/weed/s3api/s3lifecycle/bootstrap/walker_test.go index 58f73810f..4c7769de3 100644 --- a/weed/s3api/s3lifecycle/bootstrap/walker_test.go +++ b/weed/s3api/s3lifecycle/bootstrap/walker_test.go @@ -151,7 +151,7 @@ func TestWalk_NotYetDueSkipped(t *testing.T) { } snap := compileEvDriven(t, "bk", rule) mod := mustTime(t, "2024-01-01T00:00:00Z") - now := mod.AddDate(0, 0, 10) // before the 30d threshold + now := mod.Add(s3lifecycle.DaysToDuration(10)) // before the 30d threshold rec := &recorder{} if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{ diff --git a/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_default.go b/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_default.go new file mode 100644 index 000000000..731d294c2 --- /dev/null +++ b/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_default.go @@ -0,0 +1,10 @@ +//go:build !s3tests + +package dispatcher + +import "time" + +const ( + defaultDispatchTick = 5 * time.Second + defaultCheckpointTick = 30 * time.Second +) diff --git a/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_s3tests.go b/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_s3tests.go new file mode 100644 index 000000000..0d908597e --- /dev/null +++ b/weed/s3api/s3lifecycle/dispatcher/dispatch_ticks_s3tests.go @@ -0,0 +1,14 @@ +//go:build s3tests + +package dispatcher + +import "time" + +// Under the s3tests build tag the engine treats one "Day" as 10 seconds +// (util.LifeCycleInterval), so the dispatcher must run far below that to +// notice a freshly-due action inside the upstream s3-tests 30s polling +// window. Production timings live in dispatch_ticks_default.go. +const ( + defaultDispatchTick = 500 * time.Millisecond + defaultCheckpointTick = 2 * time.Second +) diff --git a/weed/s3api/s3lifecycle/dispatcher/pipeline.go b/weed/s3api/s3lifecycle/dispatcher/pipeline.go index 75027550a..6e7f153e0 100644 --- a/weed/s3api/s3lifecycle/dispatcher/pipeline.go +++ b/weed/s3api/s3lifecycle/dispatcher/pipeline.go @@ -54,14 +54,54 @@ type Pipeline struct { // EventBuffer sets the channel capacity between reader and router // goroutines. Zero = defaultEventBuffer. EventBuffer int + + // events is the input channel for the dispatch goroutine. The reader + // is the primary writer; InjectEvent allows external code (the bucket + // bootstrapper) to push synthesized events through the same router + // path. Initialized lazily by InjectEvent and Run; ready signals + // readiness to InjectEvent callers that arrive before Run. + eventsOnce sync.Once + events chan *reader.Event + eventsReady chan struct{} } +// ensureEventsChan lazily creates the events channel and the readiness +// signal so InjectEvent works whether it's called before or after Run. +func (p *Pipeline) ensureEventsChan() { + p.eventsOnce.Do(func() { + bufSize := p.EventBuffer + if bufSize <= 0 { + bufSize = defaultEventBuffer + } + p.events = make(chan *reader.Event, bufSize) + p.eventsReady = make(chan struct{}) + close(p.eventsReady) + }) +} + +// InjectEvent pushes a synthesized event onto the same input the reader +// feeds. Used by the bucket bootstrapper to backfill pre-rule entries: +// each entry becomes one *reader.Event, flows through router.Route, and +// schedules a Match with DueTime=mtime+delay. Set TsNs=0 so the cursor +// doesn't advance — the reader still resumes from its persisted position +// on restart. +func (p *Pipeline) InjectEvent(ctx context.Context, ev *reader.Event) error { + p.ensureEventsChan() + select { + case <-ctx.Done(): + return ctx.Err() + case p.events <- ev: + return nil + } +} + +// Tick defaults live in dispatch_ticks_*.go so the s3tests build can shrink +// them without touching production timings. defaultDispatchTick and +// defaultCheckpointTick are the only knobs that change per build tag. const ( - defaultDispatchTick = 5 * time.Second - defaultCheckpointTick = 30 * time.Second - defaultEventBuffer = 1024 - shutdownDrainTimeout = 30 * time.Second - shutdownSaveTimeout = 5 * time.Second + defaultEventBuffer = 1024 + shutdownDrainTimeout = 30 * time.Second + shutdownSaveTimeout = 5 * time.Second ) // shardState bundles per-shard mutable state so the single dispatch @@ -130,11 +170,8 @@ func (p *Pipeline) Run(ctx context.Context) error { minStartTsNs = 0 } - bufSize := p.EventBuffer - if bufSize <= 0 { - bufSize = defaultEventBuffer - } - events := make(chan *reader.Event, bufSize) + p.ensureEventsChan() + events := p.events rd := &reader.Reader{ BucketsPath: p.BucketsPath, ShardPredicate: func(s int) bool { @@ -153,11 +190,12 @@ func (p *Pipeline) Run(ctx context.Context) error { var wg sync.WaitGroup var readerErr error - // Reader goroutine. + // Reader goroutine. Doesn't close the events channel because external + // callers (BucketBootstrapper) also write to it; the dispatcher exits + // on runCtx cancellation rather than channel close. wg.Add(1) go func() { defer wg.Done() - defer close(events) readerErr = rd.Run(runCtx, p.FilerClient, p.ClientName, p.ClientID) if readerErr != nil && !isCtxShutdown(readerErr) { glog.Errorf("lifecycle reader: shards=%v: %v", shardIDs, readerErr) @@ -184,7 +222,6 @@ func (p *Pipeline) Run(ctx context.Context) error { defer dt.Stop() ct := time.NewTicker(checkpointTick) defer ct.Stop() - snap := p.Engine.Snapshot() drainAll := func() { drainCtx, drainCancel := context.WithTimeout(context.Background(), shutdownDrainTimeout) @@ -209,14 +246,18 @@ func (p *Pipeline) Run(ctx context.Context) error { if st == nil { continue } - if snap == nil { - snap = p.Engine.Snapshot() - } + // Always re-fetch the snapshot — caching it across events + // means an event arriving between dispatch ticks routes + // against a stale snap. With bootstrap injection, events + // often land within the same dispatch interval as the + // engine.Compile that introduced their bucket; routing + // against the prior (empty) snapshot would silently drop + // every match. Engine.Snapshot is an atomic Load. + snap := p.Engine.Snapshot() for _, m := range router.Route(snap, ev, time.Now()) { st.dispatch.Schedule.Add(m) } case <-dt.C: - snap = p.Engine.Snapshot() now := time.Now() for _, st := range states { st.dispatch.Tick(runCtx, now) diff --git a/weed/s3api/s3lifecycle/due_at.go b/weed/s3api/s3lifecycle/due_at.go index 67d4a2a53..0c27f5b68 100644 --- a/weed/s3api/s3lifecycle/due_at.go +++ b/weed/s3api/s3lifecycle/due_at.go @@ -1,6 +1,19 @@ package s3lifecycle -import "time" +import ( + "time" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// DaysToDuration converts a "days" lifecycle threshold into a wall-clock +// duration. Production builds keep one day = 24h; the s3tests build tag +// shrinks it to LifeCycleInterval (10s) so the upstream s3-tests +// expiration suite can complete inside its 30s polling window. Exported so +// tests in sibling packages can pin their math to the same scale. +func DaysToDuration(days int) time.Duration { + return time.Duration(days) * util.LifeCycleInterval +} // ComputeDueAt returns the earliest wall-clock time the (rule, kind) action // can fire for info. Returns zero time when no action can fire for this @@ -16,7 +29,7 @@ func ComputeDueAt(rule *Rule, kind ActionKind, info *ObjectInfo) time.Time { switch kind { case ActionKindAbortMPU: if info.IsMPUInit && rule.AbortMPUDaysAfterInitiation > 0 { - return info.ModTime.AddDate(0, 0, rule.AbortMPUDaysAfterInitiation) + return info.ModTime.Add(DaysToDuration(rule.AbortMPUDaysAfterInitiation)) } case ActionKindExpiredDeleteMarker: if info.IsLatest && info.IsDeleteMarker && rule.ExpiredObjectDeleteMarker && info.NumVersions == 1 { @@ -24,7 +37,7 @@ func ComputeDueAt(rule *Rule, kind ActionKind, info *ObjectInfo) time.Time { } case ActionKindExpirationDays: if info.IsLatest && !info.IsDeleteMarker && rule.ExpirationDays > 0 { - return info.ModTime.AddDate(0, 0, rule.ExpirationDays) + return info.ModTime.Add(DaysToDuration(rule.ExpirationDays)) } case ActionKindExpirationDate: if info.IsLatest && !info.IsDeleteMarker && !rule.ExpirationDate.IsZero() { @@ -36,7 +49,7 @@ func ComputeDueAt(rule *Rule, kind ActionKind, info *ObjectInfo) time.Time { if base.IsZero() { base = info.ModTime } - return base.AddDate(0, 0, rule.NoncurrentVersionExpirationDays) + return base.Add(DaysToDuration(rule.NoncurrentVersionExpirationDays)) } case ActionKindNewerNoncurrent: if !info.IsLatest && rule.NoncurrentVersionExpirationDays == 0 && rule.NewerNoncurrentVersions > 0 { diff --git a/weed/s3api/s3lifecycle/due_at_test.go b/weed/s3api/s3lifecycle/due_at_test.go index a0cd3f647..d19bb2b36 100644 --- a/weed/s3api/s3lifecycle/due_at_test.go +++ b/weed/s3api/s3lifecycle/due_at_test.go @@ -8,7 +8,7 @@ func TestComputeDueAt_ExpirationDays(t *testing.T) { rule := &Rule{Status: StatusEnabled, ExpirationDays: 30} mod := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "a", IsLatest: true, ModTime: mod} - want := mod.AddDate(0, 0, 30) + want := mod.Add(DaysToDuration(30)) if got := ComputeDueAt(rule, ActionKindExpirationDays, info); !got.Equal(want) { t.Fatalf("want %v, got %v", want, got) } @@ -59,7 +59,7 @@ func TestComputeDueAt_NoncurrentDeleteMarkerHonorsNoncurrentDays(t *testing.T) { NumVersions: 3, SuccessorModTime: successor, } - want := successor.AddDate(0, 0, 7) + want := successor.Add(DaysToDuration(7)) if got := ComputeDueAt(rule, ActionKindNoncurrentDays, info); !got.Equal(want) { t.Fatalf("want %v, got %v", want, got) } @@ -69,7 +69,7 @@ func TestComputeDueAt_NoncurrentSuccessorMtime(t *testing.T) { rule := &Rule{Status: StatusEnabled, NoncurrentVersionExpirationDays: 30} successor := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "a", IsLatest: false, SuccessorModTime: successor} - want := successor.AddDate(0, 0, 30) + want := successor.Add(DaysToDuration(30)) if got := ComputeDueAt(rule, ActionKindNoncurrentDays, info); !got.Equal(want) { t.Fatalf("want %v, got %v", want, got) } @@ -95,7 +95,7 @@ func TestComputeDueAt_MPUInit(t *testing.T) { rule := &Rule{Status: StatusEnabled, AbortMPUDaysAfterInitiation: 7} init := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: ".uploads/u1/", IsMPUInit: true, ModTime: init} - want := init.AddDate(0, 0, 7) + want := init.Add(DaysToDuration(7)) if got := ComputeDueAt(rule, ActionKindAbortMPU, info); !got.Equal(want) { t.Fatalf("want %v, got %v", want, got) } diff --git a/weed/s3api/s3lifecycle/engine/engine_test.go b/weed/s3api/s3lifecycle/engine/engine_test.go index cf28cc1e5..e61949f1e 100644 --- a/weed/s3api/s3lifecycle/engine/engine_test.go +++ b/weed/s3api/s3lifecycle/engine/engine_test.go @@ -36,7 +36,7 @@ func TestCompile_SingleRuleSingleAction(t *testing.T) { if a.Mode != ModeEventDriven { t.Fatalf("want EVENT_DRIVEN, got %v", a.Mode) } - if a.Delay != 30*24*time.Hour { + if a.Delay != s3lifecycle.DaysToDuration(30) { t.Fatalf("want 30d delay, got %v", a.Delay) } } @@ -65,9 +65,9 @@ func TestCompile_MultiAction_SiblingsHaveOwnEntries(t *testing.T) { t.Fatalf("want 3 actions, got %d", got) } wantDelays := map[time.Duration]bool{ - 90 * 24 * time.Hour: true, // ExpirationDays - 30 * 24 * time.Hour: true, // NoncurrentDays - 7 * 24 * time.Hour: true, // AbortMPU + s3lifecycle.DaysToDuration(90): true, // ExpirationDays + s3lifecycle.DaysToDuration(30): true, // NoncurrentDays + s3lifecycle.DaysToDuration(7): true, // AbortMPU } for delay, keys := range snap.originalDelayGroups { if !wantDelays[delay] { @@ -102,6 +102,15 @@ func TestCompile_BootstrapPendingIndexedButInactive(t *testing.T) { } func TestCompile_RetentionGate(t *testing.T) { + // Retention gate compares EventLogHorizon (scales with the day unit) + // against MetaLogRetention - BootstrapLookbackMin (where lookback is a + // fixed wall-clock minute count). Under the s3tests build tag the day + // unit shrinks to 10s, which collapses the margin to zero or negative + // and the gate degrades every rule. Skip there; the prod build still + // covers the gate semantics this test cares about. + if s3lifecycle.DaysToDuration(1) < time.Hour { + t.Skip("retention gate semantics require day-unit > lookback margin") + } // A 90d ExpirationDays rule with 30d retention should land in scan_only. // A 7d rule (under retention - lookback) stays event_driven. long := ruleExpDays("long", "x/", 90) @@ -115,7 +124,7 @@ func TestCompile_RetentionGate(t *testing.T) { e := New() snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: rules}}, CompileOptions{ - MetaLogRetention: 30 * 24 * time.Hour, + MetaLogRetention: s3lifecycle.DaysToDuration(30), BootstrapLookbackMin: 5 * time.Minute, PriorStates: prior, }) @@ -147,6 +156,11 @@ func TestCompile_RetentionUnboundedNeverGates(t *testing.T) { } func TestCompile_SiblingsDegradeIndependently(t *testing.T) { + // See TestCompile_RetentionGate for why the s3tests build can't honor + // the gate's absolute-time math. + if s3lifecycle.DaysToDuration(1) < time.Hour { + t.Skip("retention gate semantics require day-unit > lookback margin") + } // 90d ExpirationDays + 7d AbortMPU under 30d retention: ExpirationDays // degrades to scan_only, AbortMPU stays event_driven. The whole point // of per-action keying. @@ -164,7 +178,7 @@ func TestCompile_SiblingsDegradeIndependently(t *testing.T) { } e := New() snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{ - MetaLogRetention: 30 * 24 * time.Hour, + MetaLogRetention: s3lifecycle.DaysToDuration(30), BootstrapLookbackMin: 5 * time.Minute, PriorStates: prior, }) diff --git a/weed/s3api/s3lifecycle/engine/match_test.go b/weed/s3api/s3lifecycle/engine/match_test.go index 6009db9a6..31311fe37 100644 --- a/weed/s3api/s3lifecycle/engine/match_test.go +++ b/weed/s3api/s3lifecycle/engine/match_test.go @@ -4,7 +4,6 @@ import ( "reflect" "sort" "testing" - "time" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" ) @@ -38,11 +37,11 @@ func TestMatchOriginalWrite_DelayGroupRoutes(t *testing.T) { snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: rules}}, CompileOptions{PriorStates: activeAll("bk", rules)}) ev := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x/o"} - got30 := snap.MatchOriginalWrite(ev, 30*24*time.Hour) + got30 := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30)) if len(got30) != 1 || got30[0].ActionKind != s3lifecycle.ActionKindExpirationDays { t.Fatalf("30d sweep should match r30, got %v", got30) } - got60 := snap.MatchOriginalWrite(ev, 60*24*time.Hour) + got60 := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(60)) if len(got60) != 1 { t.Fatalf("60d sweep should match r60, got %v", got60) } @@ -53,10 +52,10 @@ func TestMatchOriginalWrite_PrefixFilter(t *testing.T) { e := New() snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{r})}) - if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "data/x"}, 30*24*time.Hour); len(got) != 0 { + if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "data/x"}, s3lifecycle.DaysToDuration(30)); len(got) != 0 { t.Fatalf("non-matching prefix should reject, got %v", got) } - if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "logs/x"}, 30*24*time.Hour); len(got) != 1 { + if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "logs/x"}, s3lifecycle.DaysToDuration(30)); len(got) != 1 { t.Fatalf("matching prefix should fire, got %v", got) } } @@ -69,11 +68,11 @@ func TestMatchOriginalWrite_MarkActiveBecomesRoutable(t *testing.T) { e := New() snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{}) ev := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x/o"} - if got := snap.MatchOriginalWrite(ev, 30*24*time.Hour); len(got) != 0 { + if got := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30)); len(got) != 0 { t.Fatalf("inactive action should not match, got %v", got) } snap.MarkActive(s3lifecycle.ActionKey{Bucket: "bk", RuleHash: s3lifecycle.RuleHash(r), ActionKind: s3lifecycle.ActionKindExpirationDays}) - if got := snap.MatchOriginalWrite(ev, 30*24*time.Hour); len(got) != 1 { + if got := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30)); len(got) != 1 { t.Fatalf("post-markActive should be routable, got %v", got) } } @@ -89,11 +88,11 @@ func TestMatchOriginalWrite_AbortMPUOnlyOnMPUInit(t *testing.T) { snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{r})}) // Non-MPU event under .uploads/ prefix: filtered out by shape gating. - got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: false}, 7*24*time.Hour) + got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: false}, s3lifecycle.DaysToDuration(7)) if len(got) != 0 { t.Fatalf("non-MPU event should be filtered, got %v", got) } - got = snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: true}, 7*24*time.Hour) + got = snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: true}, s3lifecycle.DaysToDuration(7)) if len(got) != 1 { t.Fatalf("MPU init should fire, got %v", got) } diff --git a/weed/s3api/s3lifecycle/evaluate.go b/weed/s3api/s3lifecycle/evaluate.go index 2375a96ec..9ddbb7dc9 100644 --- a/weed/s3api/s3lifecycle/evaluate.go +++ b/weed/s3api/s3lifecycle/evaluate.go @@ -30,7 +30,7 @@ func EvaluateAction(rule *Rule, kind ActionKind, info *ObjectInfo, now time.Time if !info.IsMPUInit || rule.AbortMPUDaysAfterInitiation <= 0 { return none } - due := info.ModTime.AddDate(0, 0, rule.AbortMPUDaysAfterInitiation) + due := info.ModTime.Add(DaysToDuration(rule.AbortMPUDaysAfterInitiation)) if now.Before(due) { return none } @@ -49,7 +49,7 @@ func EvaluateAction(rule *Rule, kind ActionKind, info *ObjectInfo, now time.Time if !info.IsLatest || info.IsDeleteMarker || rule.ExpirationDays <= 0 { return none } - due := info.ModTime.AddDate(0, 0, rule.ExpirationDays) + due := info.ModTime.Add(DaysToDuration(rule.ExpirationDays)) if now.Before(due) { return none } @@ -72,7 +72,7 @@ func EvaluateAction(rule *Rule, kind ActionKind, info *ObjectInfo, now time.Time if base.IsZero() { base = info.ModTime } - due := base.AddDate(0, 0, rule.NoncurrentVersionExpirationDays) + due := base.Add(DaysToDuration(rule.NoncurrentVersionExpirationDays)) if now.Before(due) { return none } diff --git a/weed/s3api/s3lifecycle/evaluate_test.go b/weed/s3api/s3lifecycle/evaluate_test.go index 99e1b08d9..76e04c68d 100644 --- a/weed/s3api/s3lifecycle/evaluate_test.go +++ b/weed/s3api/s3lifecycle/evaluate_test.go @@ -41,13 +41,13 @@ func TestEvaluateAction_ExpirationDaysBoundary(t *testing.T) { mod := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "a", IsLatest: true, ModTime: mod} - if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.AddDate(0, 0, 30).Add(-time.Nanosecond)); got.Action != ActionNone { + if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.Add(DaysToDuration(30)).Add(-time.Nanosecond)); got.Action != ActionNone { t.Fatalf("not yet due, got %v", got) } - if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.AddDate(0, 0, 30)); got.Action != ActionDeleteObject || got.RuleID != "r" { + if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.Add(DaysToDuration(30))); got.Action != ActionDeleteObject || got.RuleID != "r" { t.Fatalf("at boundary, got %v", got) } - if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.AddDate(0, 1, 0)); got.Action != ActionDeleteObject { + if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.Add(DaysToDuration(31))); got.Action != ActionDeleteObject { t.Fatalf("past due, got %v", got) } } @@ -58,7 +58,7 @@ func TestEvaluateAction_KindFiltersByDeclaredAction(t *testing.T) { rule := &Rule{Status: StatusEnabled, ExpirationDays: 30} mod := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "a", IsLatest: true, ModTime: mod} - now := mod.AddDate(0, 1, 0) + now := mod.Add(DaysToDuration(31)) if got := EvaluateAction(rule, ActionKindAbortMPU, info, now); got.Action != ActionNone { t.Fatalf("AbortMPU not declared, got %v", got) } @@ -76,7 +76,7 @@ func TestEvaluateAction_MultiActionRule_SiblingsIndependent(t *testing.T) { info := &ObjectInfo{Key: "a", IsLatest: true, ModTime: mod} // Past the 90d threshold: ExpirationDays fires. - now := mod.AddDate(0, 0, 91) + now := mod.Add(DaysToDuration(91)) if got := EvaluateAction(rule, ActionKindExpirationDays, info, now); got.Action != ActionDeleteObject { t.Fatalf("90d action should fire, got %v", got) } @@ -132,9 +132,9 @@ func TestEvaluateAction_NoncurrentDeleteMarkerExpiresViaNoncurrentDays(t *testin IsDeleteMarker: true, NumVersions: 3, SuccessorModTime: successor, - ModTime: successor.AddDate(0, 0, -1), + ModTime: successor.Add(DaysToDuration(-1)), } - if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.AddDate(0, 0, 7)); got.Action != ActionDeleteVersion { + if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.Add(DaysToDuration(7))); got.Action != ActionDeleteVersion { t.Fatalf("noncurrent delete marker should be eligible under NoncurrentDays, got %v", got) } } @@ -149,10 +149,10 @@ func TestEvaluateAction_NoncurrentVersionDays(t *testing.T) { SuccessorModTime: successor, NoncurrentIndex: idx(0), } - if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.AddDate(0, 0, 29)); got.Action != ActionNone { + if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.Add(DaysToDuration(29))); got.Action != ActionNone { t.Fatalf("not due, got %v", got) } - if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.AddDate(0, 0, 30)); got.Action != ActionDeleteVersion { + if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, successor.Add(DaysToDuration(30))); got.Action != ActionDeleteVersion { t.Fatalf("due, got %v", got) } } @@ -161,7 +161,7 @@ func TestEvaluateAction_NoncurrentDaysFallsBackToModTime(t *testing.T) { rule := &Rule{Status: StatusEnabled, NoncurrentVersionExpirationDays: 30} mod := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "a", IsLatest: false, ModTime: mod, NoncurrentIndex: idx(0)} - if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, mod.AddDate(0, 0, 30)); got.Action != ActionDeleteVersion { + if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, mod.Add(DaysToDuration(30))); got.Action != ActionDeleteVersion { t.Fatalf("expected fallback to ModTime, got %v", got) } } @@ -179,7 +179,7 @@ func TestEvaluateAction_NoncurrentDays_WithKeepN_NilIndexIsNoOp(t *testing.T) { SuccessorModTime: successor, NoncurrentIndex: nil, } - now := successor.AddDate(0, 0, 30) + now := successor.Add(DaysToDuration(30)) if got := EvaluateAction(rule, ActionKindNoncurrentDays, info, now); got.Action != ActionNone { t.Fatalf("nil index with keep-N must be no-op, got %v", got) } @@ -218,7 +218,7 @@ func TestEvaluateAction_NewerNoncurrentCountOnly(t *testing.T) { func TestEvaluateAction_NoncurrentDaysAndCount(t *testing.T) { rule := &Rule{Status: StatusEnabled, NoncurrentVersionExpirationDays: 30, NewerNoncurrentVersions: 2} successor := mustTime(t, "2024-01-01T00:00:00Z") - now := successor.AddDate(0, 0, 30) + now := successor.Add(DaysToDuration(30)) for i := 0; i < 2; i++ { info := &ObjectInfo{Key: "a", IsLatest: false, ModTime: successor, SuccessorModTime: successor, NoncurrentIndex: idx(i)} @@ -236,10 +236,10 @@ func TestEvaluateAction_AbortMultipartUpload(t *testing.T) { rule := &Rule{Status: StatusEnabled, AbortMPUDaysAfterInitiation: 7} init := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: ".uploads/u1/", IsMPUInit: true, ModTime: init} - if got := EvaluateAction(rule, ActionKindAbortMPU, info, init.AddDate(0, 0, 6)); got.Action != ActionNone { + if got := EvaluateAction(rule, ActionKindAbortMPU, info, init.Add(DaysToDuration(6))); got.Action != ActionNone { t.Fatalf("not due, got %v", got) } - if got := EvaluateAction(rule, ActionKindAbortMPU, info, init.AddDate(0, 0, 7)); got.Action != ActionAbortMultipartUpload { + if got := EvaluateAction(rule, ActionKindAbortMPU, info, init.Add(DaysToDuration(7))); got.Action != ActionAbortMultipartUpload { t.Fatalf("due, got %v", got) } } @@ -247,7 +247,7 @@ func TestEvaluateAction_AbortMultipartUpload(t *testing.T) { func TestEvaluateAction_PrefixFilter(t *testing.T) { rule := &Rule{Status: StatusEnabled, Prefix: "logs/", ExpirationDays: 1} mod := mustTime(t, "2024-01-01T00:00:00Z") - now := mod.AddDate(0, 0, 10) + now := mod.Add(DaysToDuration(10)) if got := EvaluateAction(rule, ActionKindExpirationDays, &ObjectInfo{Key: "data/x", IsLatest: true, ModTime: mod}, now); got.Action != ActionNone { t.Fatalf("prefix mismatch should reject, got %v", got) } @@ -259,7 +259,7 @@ func TestEvaluateAction_PrefixFilter(t *testing.T) { func TestEvaluateAction_TagFilter(t *testing.T) { rule := &Rule{Status: StatusEnabled, ExpirationDays: 1, FilterTags: map[string]string{"env": "prod", "tier": "cold"}} mod := mustTime(t, "2024-01-01T00:00:00Z") - now := mod.AddDate(0, 0, 10) + now := mod.Add(DaysToDuration(10)) info := &ObjectInfo{Key: "a", IsLatest: true, ModTime: mod} if got := EvaluateAction(rule, ActionKindExpirationDays, info, now); got.Action != ActionNone { @@ -278,7 +278,7 @@ func TestEvaluateAction_TagFilter(t *testing.T) { func TestEvaluateAction_SizeFilter(t *testing.T) { rule := &Rule{Status: StatusEnabled, ExpirationDays: 1, FilterSizeGreaterThan: 100, FilterSizeLessThan: 1000} mod := mustTime(t, "2024-01-01T00:00:00Z") - now := mod.AddDate(0, 0, 10) + now := mod.Add(DaysToDuration(10)) cases := []struct { size int64 want Action @@ -301,7 +301,7 @@ func TestEvaluateAction_EmptyPrefixMatchesAll(t *testing.T) { rule := &Rule{Status: StatusEnabled, ExpirationDays: 1, Prefix: ""} mod := mustTime(t, "2024-01-01T00:00:00Z") info := &ObjectInfo{Key: "deeply/nested/path/x", IsLatest: true, ModTime: mod} - if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.AddDate(0, 0, 10)); got.Action != ActionDeleteObject { + if got := EvaluateAction(rule, ActionKindExpirationDays, info, mod.Add(DaysToDuration(10))); got.Action != ActionDeleteObject { t.Fatalf("empty prefix should match, got %v", got) } } @@ -316,7 +316,7 @@ func TestEvaluateAction_MPUInitDoesNotFireNoncurrent(t *testing.T) { NoncurrentVersionExpirationDays: 7, NewerNoncurrentVersions: 3, } - init := time.Now().AddDate(0, 0, -30) + init := time.Now().Add(-DaysToDuration(30)) info := &ObjectInfo{Key: "logs/foo.txt", IsMPUInit: true, ModTime: init} for _, kind := range []ActionKind{ActionKindNoncurrentDays, ActionKindNewerNoncurrent} { diff --git a/weed/s3api/s3lifecycle/event_log_horizon.go b/weed/s3api/s3lifecycle/event_log_horizon.go index 7c909cc74..7a62d4fa6 100644 --- a/weed/s3api/s3lifecycle/event_log_horizon.go +++ b/weed/s3api/s3lifecycle/event_log_horizon.go @@ -11,19 +11,18 @@ func EventLogHorizon(rule *Rule, kind ActionKind) time.Duration { if rule == nil { return 0 } - const day = 24 * time.Hour switch kind { case ActionKindExpirationDays: if rule.ExpirationDays > 0 { - return time.Duration(rule.ExpirationDays) * day + return DaysToDuration(rule.ExpirationDays) } case ActionKindNoncurrentDays: if rule.NoncurrentVersionExpirationDays > 0 { - return time.Duration(rule.NoncurrentVersionExpirationDays) * day + return DaysToDuration(rule.NoncurrentVersionExpirationDays) } case ActionKindAbortMPU: if rule.AbortMPUDaysAfterInitiation > 0 { - return time.Duration(rule.AbortMPUDaysAfterInitiation) * day + return DaysToDuration(rule.AbortMPUDaysAfterInitiation) } case ActionKindNewerNoncurrent: if rule.NewerNoncurrentVersions > 0 && rule.NoncurrentVersionExpirationDays == 0 { diff --git a/weed/s3api/s3lifecycle/event_log_horizon_test.go b/weed/s3api/s3lifecycle/event_log_horizon_test.go index 3d8a58db1..a254fcbe9 100644 --- a/weed/s3api/s3lifecycle/event_log_horizon_test.go +++ b/weed/s3api/s3lifecycle/event_log_horizon_test.go @@ -14,9 +14,9 @@ func TestEventLogHorizon_PerActionIsIndependent(t *testing.T) { kind ActionKind want time.Duration }{ - {ActionKindExpirationDays, 90 * 24 * time.Hour}, - {ActionKindAbortMPU, 7 * 24 * time.Hour}, - {ActionKindNoncurrentDays, 30 * 24 * time.Hour}, + {ActionKindExpirationDays, DaysToDuration(90)}, + {ActionKindAbortMPU, DaysToDuration(7)}, + {ActionKindNoncurrentDays, DaysToDuration(30)}, {ActionKindExpirationDate, 0}, } for _, c := range cases { @@ -44,7 +44,7 @@ func TestEventLogHorizon_NewerNoncurrentWithDaysIsZero(t *testing.T) { t.Fatalf("NEWER_NONCURRENT not declared when paired with days, got %v", got) } // The actual action this rule declares is NONCURRENT_DAYS — verify horizon there. - if got := EventLogHorizon(rule, ActionKindNoncurrentDays); got != 30*24*time.Hour { + if got := EventLogHorizon(rule, ActionKindNoncurrentDays); got != DaysToDuration(30) { t.Fatalf("NoncurrentDays horizon, want 30d, got %v", got) } } diff --git a/weed/s3api/s3lifecycle/min_trigger_age.go b/weed/s3api/s3lifecycle/min_trigger_age.go index 56779ae09..23610bf08 100644 --- a/weed/s3api/s3lifecycle/min_trigger_age.go +++ b/weed/s3api/s3lifecycle/min_trigger_age.go @@ -9,19 +9,18 @@ func MinTriggerAge(rule *Rule, kind ActionKind) time.Duration { if rule == nil { return 0 } - const day = 24 * time.Hour switch kind { case ActionKindExpirationDays: if rule.ExpirationDays > 0 { - return time.Duration(rule.ExpirationDays) * day + return DaysToDuration(rule.ExpirationDays) } case ActionKindNoncurrentDays: if rule.NoncurrentVersionExpirationDays > 0 { - return time.Duration(rule.NoncurrentVersionExpirationDays) * day + return DaysToDuration(rule.NoncurrentVersionExpirationDays) } case ActionKindAbortMPU: if rule.AbortMPUDaysAfterInitiation > 0 { - return time.Duration(rule.AbortMPUDaysAfterInitiation) * day + return DaysToDuration(rule.AbortMPUDaysAfterInitiation) } } return 0 diff --git a/weed/s3api/s3lifecycle/min_trigger_age_test.go b/weed/s3api/s3lifecycle/min_trigger_age_test.go index e6ce3f25e..2da8e8ba6 100644 --- a/weed/s3api/s3lifecycle/min_trigger_age_test.go +++ b/weed/s3api/s3lifecycle/min_trigger_age_test.go @@ -15,9 +15,9 @@ func TestMinTriggerAge_PerKind(t *testing.T) { kind ActionKind want time.Duration }{ - {ActionKindExpirationDays, 30 * 24 * time.Hour}, - {ActionKindNoncurrentDays, 7 * 24 * time.Hour}, - {ActionKindAbortMPU, 14 * 24 * time.Hour}, + {ActionKindExpirationDays, DaysToDuration(30)}, + {ActionKindNoncurrentDays, DaysToDuration(7)}, + {ActionKindAbortMPU, DaysToDuration(14)}, {ActionKindExpirationDate, 0}, {ActionKindNewerNoncurrent, 0}, {ActionKindExpiredDeleteMarker, 0}, diff --git a/weed/s3api/s3lifecycle/router/router_test.go b/weed/s3api/s3lifecycle/router/router_test.go index 592485484..eb9a96a32 100644 --- a/weed/s3api/s3lifecycle/router/router_test.go +++ b/weed/s3api/s3lifecycle/router/router_test.go @@ -111,7 +111,7 @@ func TestRouteFreshObjectSchedulesInFuture(t *testing.T) { if len(matches) != 1 { t.Fatalf("expected 1 match (scheduled), got %v", matches) } - if !matches[0].DueTime.After(now.Add(6 * 24 * time.Hour)) { + if !matches[0].DueTime.After(now.Add(s3lifecycle.DaysToDuration(6))) { t.Fatalf("DueTime=%v, want ~7 days from now", matches[0].DueTime) } } diff --git a/weed/s3api/s3lifecycle/scheduler/bootstrap.go b/weed/s3api/s3lifecycle/scheduler/bootstrap.go new file mode 100644 index 000000000..ee66bf993 --- /dev/null +++ b/weed/s3api/s3lifecycle/scheduler/bootstrap.go @@ -0,0 +1,154 @@ +package scheduler + +import ( + "context" + "fmt" + "strings" + "sync" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader" +) + +// EventInjector is the bootstrap-side hook into the dispatcher pipeline. +// One implementation routes events to the right per-shard pipeline; the +// shell's single-pipeline path passes pipeline.InjectEvent directly. +type EventInjector interface { + InjectEvent(ctx context.Context, ev *reader.Event) error +} + +// BucketBootstrapper backfills already-existing entries when a freshly-PUT +// rule's bucket appears in the engine. The reader-driven path only sees +// meta-log events created after the rule lands; without this walk, +// objects PUT before the rule would never expire. +// +// Per bucket: one one-shot goroutine that lists every entry under +// /buckets/ and synthesizes a *reader.Event for each one. The +// pipeline's existing router.Route + Schedule machinery handles the rest: +// currently-due matches fire on the next dispatch tick, and not-yet-due +// matches sit in the per-shard schedule until their DueTime arrives. +// +// Synthesized events carry TsNs=0 so dispatcher.advance is a no-op for +// them — the reader still resumes from its persisted cursor on restart. +// +// Bucket completion is in-memory per process; a fresh worker run walks +// each bucket once on first refresh. +type BucketBootstrapper struct { + FilerClient filer_pb.SeaweedFilerClient + BucketsPath string + Injector EventInjector + + mu sync.Mutex + known map[string]bool +} + +// KickOffNew launches a one-shot walker goroutine for every bucket +// in `buckets` that hasn't been seen before. +func (b *BucketBootstrapper) KickOffNew(ctx context.Context, buckets []string) { + if b.Injector == nil { + return + } + b.mu.Lock() + if b.known == nil { + b.known = map[string]bool{} + } + fresh := make([]string, 0, len(buckets)) + for _, bucket := range buckets { + if b.known[bucket] { + continue + } + b.known[bucket] = true + fresh = append(fresh, bucket) + } + b.mu.Unlock() + + for _, bucket := range fresh { + bucket := bucket + go b.walkBucket(ctx, bucket) + } +} + +func (b *BucketBootstrapper) walkBucket(ctx context.Context, bucket string) { + root := strings.TrimSuffix(b.BucketsPath, "/") + "/" + bucket + glog.V(0).Infof("lifecycle bootstrap: starting walk for bucket %s (root=%s)", bucket, root) + count := 0 + if err := walkBucketDir(ctx, b.FilerClient, root, root, func(entry *filer_pb.Entry, key string) error { + ev := &reader.Event{ + // TsNs=0 sentinel: dispatcher.advance treats <=0 as no-op, + // so the reader's persisted cursor isn't ratcheted forward + // past meta-log events that haven't been processed yet. + TsNs: 0, + Bucket: bucket, + Key: key, + ShardID: s3lifecycle.ShardID(bucket, key), + NewEntry: entry, + } + count++ + return b.Injector.InjectEvent(ctx, ev) + }); err != nil { + if ctx.Err() == nil { + glog.V(0).Infof("lifecycle bootstrap %s: %v", bucket, err) + } + return + } + glog.V(0).Infof("lifecycle bootstrap: bucket %s injected %d entries", bucket, count) +} + +// walkBucketDir lists every file under dir recursively and invokes cb +// with the filer entry plus its bucket-relative key. MPU init dirs at +// .uploads/ are emitted as a single (directory-shaped) entry so the +// router's MPU detection fires; deeper directories recurse. +func walkBucketDir(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot string, cb func(entry *filer_pb.Entry, key string) error) error { + var children []*filer_pb.Entry + if err := filer_pb.SeaweedList(ctx, client, dir, "", func(e *filer_pb.Entry, _ bool) error { + children = append(children, e) + return nil + }, "", false, 0); err != nil { + return fmt.Errorf("list %s: %w", dir, err) + } + for _, entry := range children { + if entry == nil || entry.Attributes == nil { + continue + } + full := dir + "/" + entry.Name + key := strings.TrimPrefix(full, bucketRoot+"/") + + if entry.IsDirectory { + if isMPUInitDir(key, entry) { + if err := cb(entry, key); err != nil { + return err + } + continue + } + if err := walkBucketDir(ctx, client, full, bucketRoot, cb); err != nil { + return err + } + continue + } + if err := cb(entry, key); err != nil { + return err + } + } + return nil +} + +// isMPUInitDir mirrors router.mpuInitInfo: a directory at .uploads/ +// carrying the destination key in Extended is the MPU init record. The +// router helper is package-private so this is duplicated rather than +// adding a public extraction API just for this caller. +func isMPUInitDir(key string, entry *filer_pb.Entry) bool { + uploadsPrefix := s3_constants.MultipartUploadsFolder + "/" + if !strings.HasPrefix(key, uploadsPrefix) { + return false + } + rest := key[len(uploadsPrefix):] + if rest == "" || strings.ContainsRune(rest, '/') { + return false + } + v, ok := entry.Extended[s3_constants.ExtMultipartObjectKey] + return ok && len(v) > 0 +} + diff --git a/weed/s3api/s3lifecycle/scheduler/refresh_default.go b/weed/s3api/s3lifecycle/scheduler/refresh_default.go new file mode 100644 index 000000000..5c0a4330c --- /dev/null +++ b/weed/s3api/s3lifecycle/scheduler/refresh_default.go @@ -0,0 +1,7 @@ +//go:build !s3tests + +package scheduler + +import "time" + +const defaultRefreshInterval = 5 * time.Minute diff --git a/weed/s3api/s3lifecycle/scheduler/refresh_s3tests.go b/weed/s3api/s3lifecycle/scheduler/refresh_s3tests.go new file mode 100644 index 000000000..219749122 --- /dev/null +++ b/weed/s3api/s3lifecycle/scheduler/refresh_s3tests.go @@ -0,0 +1,9 @@ +//go:build s3tests + +package scheduler + +import "time" + +// Under the s3tests build tag rules are PUT and expected to fire within +// seconds, so the engine snapshot is rebuilt aggressively. +const defaultRefreshInterval = 2 * time.Second diff --git a/weed/s3api/s3lifecycle/scheduler/scheduler.go b/weed/s3api/s3lifecycle/scheduler/scheduler.go index 803a75c1c..c4571b49b 100644 --- a/weed/s3api/s3lifecycle/scheduler/scheduler.go +++ b/weed/s3api/s3lifecycle/scheduler/scheduler.go @@ -15,9 +15,10 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader" ) +// defaultRefreshInterval lives in refresh_*.go so the s3tests build can +// shrink the engine-rebuild cadence to seconds. const ( - defaultRefreshInterval = 5 * time.Minute - defaultRetryBackoff = 5 * time.Second + defaultRetryBackoff = 5 * time.Second ) // Scheduler runs N pipeline goroutines, one per worker, each owning a @@ -62,7 +63,49 @@ func (s *Scheduler) Run(ctx context.Context) error { retry = defaultRetryBackoff } - s.refreshEngine(ctx) + // Build all pipelines up front and remember which shard each owns. + // Doing this before Run lets the bootstrap injector route an event + // to the right pipeline by shard, and lets InjectEvent's lazy events + // channel be primed before the per-bucket walker starts pushing. + type pipelineSlot struct { + pipeline *dispatcher.Pipeline + shards []int + } + var slots []*pipelineSlot + pipelinesByShard := make(map[int]*dispatcher.Pipeline) + for i := 0; i < workers; i++ { + shardSet := AssignShards(i, workers) + if len(shardSet) == 0 { + continue + } + slot := &pipelineSlot{ + pipeline: &dispatcher.Pipeline{ + Shards: shardSet, + BucketsPath: s.BucketsPath, + Engine: s.Engine, + Persister: s.Persister, + Client: s.Client, + FilerClient: s.FilerClient, + ClientID: s.ClientID, + ClientName: fmt.Sprintf("%s-w%02d", s.ClientName, i), + DispatchTick: s.DispatchTick, + CheckpointTick: s.CheckpointTick, + }, + shards: shardSet, + } + slots = append(slots, slot) + for _, sh := range shardSet { + pipelinesByShard[sh] = slot.pipeline + } + } + + bs := &BucketBootstrapper{ + FilerClient: s.FilerClient, + BucketsPath: s.BucketsPath, + Injector: pipelineFanout(pipelinesByShard), + } + + s.refreshEngine(ctx, bs) var wg sync.WaitGroup wg.Add(1) @@ -75,18 +118,15 @@ func (s *Scheduler) Run(ctx context.Context) error { case <-ctx.Done(): return case <-t.C: - s.refreshEngine(ctx) + s.refreshEngine(ctx, bs) } } }() - for i := 0; i < workers; i++ { - shards := AssignShards(i, workers) - if len(shards) == 0 { - continue - } + for _, slot := range slots { + slot := slot wg.Add(1) - go func(idx int, shardSet []int) { + go func() { defer wg.Done() for { select { @@ -94,20 +134,8 @@ func (s *Scheduler) Run(ctx context.Context) error { return default: } - p := &dispatcher.Pipeline{ - Shards: shardSet, - BucketsPath: s.BucketsPath, - Engine: s.Engine, - Persister: s.Persister, - Client: s.Client, - FilerClient: s.FilerClient, - ClientID: s.ClientID, - ClientName: fmt.Sprintf("%s-w%02d", s.ClientName, idx), - DispatchTick: s.DispatchTick, - CheckpointTick: s.CheckpointTick, - } - if err := p.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { - glog.Warningf("lifecycle scheduler: worker=%d shards=%v: %v", idx, shardSet, err) + if err := slot.pipeline.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + glog.Warningf("lifecycle scheduler: shards=%v: %v", slot.shards, err) } select { case <-ctx.Done(): @@ -115,14 +143,14 @@ func (s *Scheduler) Run(ctx context.Context) error { case <-time.After(retry): } } - }(i, shards) + }() } wg.Wait() return nil } -func (s *Scheduler) refreshEngine(ctx context.Context) { +func (s *Scheduler) refreshEngine(ctx context.Context, bs *BucketBootstrapper) { inputs, parseErrors, err := LoadCompileInputs(ctx, s.FilerClient, s.BucketsPath) if err != nil { glog.Warningf("lifecycle scheduler: refresh engine: %v", err) @@ -132,6 +160,32 @@ func (s *Scheduler) refreshEngine(ctx context.Context) { glog.Warningf("lifecycle scheduler: malformed config in bucket %s: %v", pe.Bucket, pe.Err) } s.Engine.Compile(inputs, engine.CompileOptions{PriorStates: AllActivePriorStates(inputs)}) + if bs != nil { + buckets := make([]string, 0, len(inputs)) + for _, in := range inputs { + buckets = append(buckets, in.Bucket) + } + bs.KickOffNew(ctx, buckets) + } +} + +// pipelineFanout routes a synthesized event to the pipeline that owns +// its shard. Returned as an EventInjector so the bootstrapper doesn't +// know about pipelines. +type pipelineFanout map[int]*dispatcher.Pipeline + +func (f pipelineFanout) InjectEvent(ctx context.Context, ev *reader.Event) error { + if ev == nil { + return nil + } + p, ok := f[ev.ShardID] + if !ok { + // Shard isn't covered by any pipeline in this scheduler — nothing + // to do. This shouldn't happen with AssignShards covering [0, ShardCount), + // but stay tolerant if a future change introduces gaps. + return nil + } + return p.InjectEvent(ctx, ev) } // AssignShards returns the contiguous shard slice for worker idx of total. diff --git a/weed/shell/command_s3_lifecycle_run_shard.go b/weed/shell/command_s3_lifecycle_run_shard.go index a39eb61e7..ace7f489f 100644 --- a/weed/shell/command_s3_lifecycle_run_shard.go +++ b/weed/shell/command_s3_lifecycle_run_shard.go @@ -67,6 +67,7 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i eventBudget := fs.Int("events", 1000, "max in-shard events to process before returning (0 = unbounded; counts only events that pass the shard filter)") dispatchTick := fs.Duration("dispatch", 5*time.Second, "dispatcher tick cadence") checkpointTick := fs.Duration("checkpoint", 30*time.Second, "cursor checkpoint cadence") + refreshInterval := fs.Duration("refresh", 5*time.Minute, "interval for rebuilding the engine snapshot from filer-backed bucket configs; 0 = compile once at startup") runtime := fs.Duration("runtime", 0, "wall-clock cap on the run; 0 = no timeout. -events alone can hang on quiet shards") if err := fs.Parse(args); err != nil { return err @@ -101,26 +102,7 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i // Run the whole pipeline inside one WithFilerClient so the reader's // SubscribeMetadata stream and the persister share a single connection. return env.WithFilerClient(true, func(filerClient filer_pb.SeaweedFilerClient) error { - inputs, parseErrors, err := scheduler.LoadCompileInputs(context.Background(), filerClient, bucketsPath) - if err != nil { - return fmt.Errorf("load lifecycle configs: %w", err) - } - for i, pe := range parseErrors { - if i < 3 { - fmt.Fprintf(writer, "warning: %s: %v\n", pe.Bucket, pe.Err) - } - } - if extra := len(parseErrors) - 3; extra > 0 { - fmt.Fprintf(writer, "warning: %d additional bucket(s) had malformed lifecycle config\n", extra) - } - if len(inputs) == 0 { - fmt.Fprintln(writer, "no buckets with enabled lifecycle rules found") - return nil - } - fmt.Fprintf(writer, "loaded lifecycle for %d bucket(s)\n", len(inputs)) - eng := engine.New() - eng.Compile(inputs, engine.CompileOptions{PriorStates: scheduler.AllActivePriorStates(inputs)}) pipeline := &dispatcher.Pipeline{ Shards: shards, @@ -136,6 +118,51 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i EventBudget: *eventBudget, } + bsr := &scheduler.BucketBootstrapper{ + FilerClient: filerClient, + BucketsPath: bucketsPath, + Injector: pipeline, + } + bootstrapCtx, bootstrapCancel := context.WithCancel(context.Background()) + defer bootstrapCancel() + + compile := func(initial bool) { + inputs, parseErrors, err := scheduler.LoadCompileInputs(context.Background(), filerClient, bucketsPath) + if err != nil { + if initial { + fmt.Fprintf(writer, "warning: load lifecycle configs: %v\n", err) + } + return + } + if initial { + for i, pe := range parseErrors { + if i < 3 { + fmt.Fprintf(writer, "warning: %s: %v\n", pe.Bucket, pe.Err) + } + } + if extra := len(parseErrors) - 3; extra > 0 { + fmt.Fprintf(writer, "warning: %d additional bucket(s) had malformed lifecycle config\n", extra) + } + if len(inputs) == 0 { + fmt.Fprintln(writer, "no buckets with enabled lifecycle rules at startup; will refresh and pick them up as they're added") + } else { + fmt.Fprintf(writer, "loaded lifecycle for %d bucket(s)\n", len(inputs)) + } + } + eng.Compile(inputs, engine.CompileOptions{PriorStates: scheduler.AllActivePriorStates(inputs)}) + // First time we see a bucket, spin up a one-shot walker that + // lists existing entries and synthesizes events. The reader- + // driven path only sees events created after the rule lands, + // so without this backfill objects PUT before the rule (the + // s3-tests scenario) would never expire. + buckets := make([]string, 0, len(inputs)) + for _, in := range inputs { + buckets = append(buckets, in.Bucket) + } + bsr.KickOffNew(bootstrapCtx, buckets) + } + compile(true) + var ctx context.Context var cancel context.CancelFunc if *runtime > 0 { @@ -145,7 +172,24 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i } defer cancel() - fmt.Fprintf(writer, "running shards %s (event budget=%d, runtime=%s)…\n", formatShardLabel(shards), *eventBudget, *runtime) + // Periodic engine rebuild so rules added (or disabled) after startup + // land in the snapshot the dispatcher reads on its next tick. + if *refreshInterval > 0 { + go func() { + t := time.NewTicker(*refreshInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + compile(false) + } + } + }() + } + + fmt.Fprintf(writer, "running shards %s (event budget=%d, runtime=%s, refresh=%s)…\n", formatShardLabel(shards), *eventBudget, *runtime, *refreshInterval) if err := pipeline.Run(ctx); err != nil { return fmt.Errorf("pipeline: %w", err) }