Files
Chris LuandGitHub 3f4cb6d2fb feat(s3/lifecycle/engine): daily-replay view surface (Phase 4 engine) (#9447)
* feat(s3/lifecycle/engine): daily-replay view surface (Phase 4 engine)

Adds the engine-side API the new daily-replay worker reaches for:
per-view snapshot construction (RulesForShard, RecoveryView), the two
cursor hashes that gate recovery (ReplayContentHash, PromotedHash),
and the cursor sliding-window helper (MaxEffectiveTTL). CurrentSnapshot
is a stub keyed on a package-level atomic that the worker startup wiring
populates.

Views return new *Snapshot instances holding cloned *CompiledAction
values so per-clone active/Mode never leak across partitions. Replay
clones force Mode=ModeEventDriven to rehabilitate any persistent
ModeScanOnly carried over from PriorState; walk and recovery clones
preserve Mode as-is. Disabled actions are excluded from all views.

No production caller is wired here — Phase 4's walker/dailyrun
integration is the follow-up. dailyrun's local helpers
(localReplayContentHash, localMaxEffectiveTTL) become one-line
redirects to these exports.

API surface:
- CurrentSnapshot() *Snapshot — stub until Phase 4 wiring.
- SetCurrentEngine(*Engine) — Phase 4 wiring entry point.
- Snapshot.RulesForShard(shardID, retentionWindow) (replay, walk *Snapshot)
- RecoveryView(s *Snapshot) *Snapshot — force-active over the full set.
- ReplayContentHash(s *Snapshot) [32]byte — partition-independent.
- PromotedHash(s *Snapshot, retentionWindow) [32]byte — partition-flip.
- MaxEffectiveTTL(s *Snapshot) time.Duration — over active replay only.

30 unit tests covering clone isolation, Mode rewrite, partition
membership including the multi-action-kind XML rule split,
RecoveryView activating pre-BootstrapComplete actions,
ReplayContentHash partition-independence, PromotedHash sensitivity to
promotion in either direction, MaxEffectiveTTL aggregation. Build +
race-tests green.

* refactor(s3/lifecycle/engine): consolidate hash helpers; clarify shardID semantics

Addresses PR #9447 review feedback. Three medium-priority items from
gemini, all code-quality refinements (no behavior change):

1. Duplicated sort comparator between ReplayContentHash and
   PromotedHash. Extract sortHashItems shared helper so the two
   hashes use the same ordering by construction — if one drifted, the
   cursor could see a spurious "rule changed" on a no-op snapshot
   rebuild.

2. Duplicated writeField/writeInt closures. Extract hashWriter struct
   holding the sha256 running hash + lenbuf, with method helpers.
   Same allocation profile (one Hash, one tiny stack buffer per
   helper); just deduplicates ~20 lines.

3. shardID parameter on RulesForShard is unused. Per the design's
   open question, every shard sees every rule today (shard filter
   runs at the entry-iteration site, not view construction). Keep
   the parameter for API stability — removing it now would force
   a breaking change when bucket-shard ownership lands — and update
   the doc comment to explain why it's reserved.

go build ./... clean; engine test suite green.
2026-05-11 18:07:54 -07:00

326 lines
12 KiB
Go

package engine
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// buildSnapshotForViews compiles a snapshot with a single bucket containing
// one rule per supplied factory, marking every action bootstrap_complete so
// the base actions are active. Tests can then call RulesForShard and assert
// the per-view clones come out independent.
func buildSnapshotForViews(t *testing.T, bucket string, rules ...*s3lifecycle.Rule) *Snapshot {
t.Helper()
prior := map[s3lifecycle.ActionKey]PriorState{}
for _, r := range rules {
rh := s3lifecycle.RuleHash(r)
for _, k := range s3lifecycle.RuleActionKinds(r) {
prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: rh, ActionKind: k}] = PriorState{
BootstrapComplete: true,
}
}
}
return New().Compile(
[]CompileInput{{Bucket: bucket, Rules: rules}},
CompileOptions{PriorStates: prior},
)
}
func TestCurrentSnapshot_NilWhenNoEngineRegistered(t *testing.T) {
// Save and restore the package-level pointer so tests stay isolated.
prev := currentEngine.Load()
t.Cleanup(func() { currentEngine.Store(prev) })
SetCurrentEngine(nil)
assert.Nil(t, CurrentSnapshot())
}
func TestCurrentSnapshot_ReturnsRegisteredEngineSnapshot(t *testing.T) {
prev := currentEngine.Load()
t.Cleanup(func() { currentEngine.Store(prev) })
e := New()
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
snap := e.Compile([]CompileInput{{Bucket: "b", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{})
SetCurrentEngine(e)
assert.Same(t, snap, CurrentSnapshot())
}
func TestRulesForShard_NilSnapshot(t *testing.T) {
var s *Snapshot
replay, walk := s.RulesForShard(0, 30*24*time.Hour)
assert.Nil(t, replay)
assert.Nil(t, walk)
}
func TestRulesForShard_ReplayPartitionMembership(t *testing.T) {
// ExpirationDays / NoncurrentDays / AbortMPU below retentionWindow all
// land in replay; the replay map carries clones with active=true and
// Mode=ModeEventDriven.
rule := &s3lifecycle.Rule{
ID: "all-replay",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 3,
NoncurrentVersionExpirationDays: 5,
AbortMPUDaysAfterInitiation: 2,
}
snap := buildSnapshotForViews(t, "b1", rule)
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
require.NotNil(t, replay, "all three replay-eligible kinds should land in replay")
assert.Nil(t, walk, "no walker kinds, no scan_only promotion, walk should be empty")
assert.Len(t, replay.actions, 3)
for k, a := range replay.actions {
assert.True(t, a.IsActive(), "replay clone must be active for kind %v", k.ActionKind)
assert.Equal(t, ModeEventDriven, a.Mode, "replay Mode must be ModeEventDriven for kind %v", k.ActionKind)
}
}
func TestRulesForShard_WalkOnlyKindsRouteToWalk(t *testing.T) {
// ExpirationDate, ExpiredObjectDeleteMarker, NewerNoncurrent are walker-
// only and must land in walk regardless of retentionWindow.
when := time.Now().Add(24 * time.Hour)
rule := &s3lifecycle.Rule{
ID: "walker-mix",
Status: s3lifecycle.StatusEnabled,
ExpirationDate: when,
ExpiredObjectDeleteMarker: true,
NewerNoncurrentVersions: 2,
}
snap := buildSnapshotForViews(t, "b1", rule)
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
assert.Nil(t, replay, "walker-only rule has no replay partition")
require.NotNil(t, walk)
assert.Len(t, walk.actions, 3, "all three walker kinds should land in walk")
for _, a := range walk.actions {
assert.True(t, a.IsActive())
}
}
func TestRulesForShard_MultiActionXMLRuleSplits(t *testing.T) {
// A single XML rule that compiles to ExpirationDays + NewerNoncurrent
// must split: ExpirationDays → replay, NewerNoncurrent → walk.
rule := &s3lifecycle.Rule{
ID: "split",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 3,
NewerNoncurrentVersions: 5,
}
snap := buildSnapshotForViews(t, "b1", rule)
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
require.NotNil(t, replay)
require.NotNil(t, walk)
assert.Len(t, replay.actions, 1)
assert.Len(t, walk.actions, 1)
for k := range replay.actions {
assert.Equal(t, s3lifecycle.ActionKindExpirationDays, k.ActionKind)
}
for k := range walk.actions {
assert.Equal(t, s3lifecycle.ActionKindNewerNoncurrent, k.ActionKind)
}
}
func TestRulesForShard_ScanOnlyPromotionRoutesToWalk(t *testing.T) {
// A replay-eligible kind whose TTL exceeds retentionWindow promotes to
// walk. The clone's active=true is set; Mode is preserved as-is from
// the base (the walker only rejects ModeDisabled).
rule := &s3lifecycle.Rule{
ID: "long",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 90,
AbortMPUDaysAfterInitiation: 1,
}
snap := buildSnapshotForViews(t, "b1", rule)
// Retention window of 30 days promotes ExpirationDays(90d) to walk;
// AbortMPU(1d) stays in replay.
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
require.NotNil(t, replay)
require.NotNil(t, walk)
assert.Len(t, replay.actions, 1)
assert.Len(t, walk.actions, 1)
for k, a := range replay.actions {
assert.Equal(t, s3lifecycle.ActionKindAbortMPU, k.ActionKind)
assert.Equal(t, ModeEventDriven, a.Mode)
}
for k := range walk.actions {
assert.Equal(t, s3lifecycle.ActionKindExpirationDays, k.ActionKind)
}
}
func TestRulesForShard_ReplayRewritesScanOnlyMode(t *testing.T) {
// A base action carrying Mode=ModeScanOnly via PriorState must still
// land in replay as ModeEventDriven once retention rehabilitates its
// TTL. router.Route gates on ModeEventDriven, so this is the
// rehabilitation contract.
rule := &s3lifecycle.Rule{
ID: "rehab",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 5,
}
rh := s3lifecycle.RuleHash(rule)
key := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
snap := New().Compile(
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
CompileOptions{
PriorStates: map[s3lifecycle.ActionKey]PriorState{
// Persistent ModeScanOnly carried over from a prior compile.
key: {BootstrapComplete: true, Mode: ModeScanOnly},
},
},
)
// Verify the base really did keep ModeScanOnly (Compile preserves it).
require.Equal(t, ModeScanOnly, snap.actions[key].Mode)
replay, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
require.NotNil(t, replay)
clone := replay.actions[key]
require.NotNil(t, clone)
assert.Equal(t, ModeEventDriven, clone.Mode, "replay must force ModeEventDriven on the clone")
assert.True(t, clone.IsActive())
}
func TestRulesForShard_DisabledRuleExcludedFromBothViews(t *testing.T) {
rule := &s3lifecycle.Rule{ID: "off", Status: s3lifecycle.StatusDisabled, ExpirationDays: 7}
snap := buildSnapshotForViews(t, "b1", rule)
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
assert.Nil(t, replay)
assert.Nil(t, walk)
}
func TestRulesForShard_ClonesAreIndependentOfBase(t *testing.T) {
// Toggling active on a view clone must never propagate to the base
// action — that's the whole point of cloning per view.
rule := &s3lifecycle.Rule{ID: "iso", Status: s3lifecycle.StatusEnabled, ExpirationDays: 3}
snap := buildSnapshotForViews(t, "b1", rule)
key := s3lifecycle.ActionKey{
Bucket: "b1",
RuleHash: s3lifecycle.RuleHash(rule),
ActionKind: s3lifecycle.ActionKindExpirationDays,
}
base := snap.Action(key)
require.NotNil(t, base)
baseActiveBefore := base.IsActive()
replay, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
require.NotNil(t, replay)
clone := replay.actions[key]
require.NotNil(t, clone)
assert.NotSame(t, base, clone, "clone must be a distinct *CompiledAction")
// Force the clone inactive (write directly to its atomic so we exercise
// the active bit boundary, not just markActive's setter).
clone.engineState.Store(engineStateInactive)
assert.False(t, clone.IsActive())
assert.Equal(t, baseActiveBefore, base.IsActive(),
"toggling the clone must not perturb the base's active bit")
}
func TestRulesForShard_ZeroRetentionRoutesAllReplayKindsToWalk(t *testing.T) {
// retentionWindow == 0 (no proven retention) must route every replay-
// eligible action into walk — the partition is "ttl > 0 && ttl <=
// retentionWindow", and 0 fails the upper bound. This is the safe
// default before the meta-log horizon is known.
rule := &s3lifecycle.Rule{
ID: "any",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 1,
}
snap := buildSnapshotForViews(t, "b1", rule)
replay, walk := snap.RulesForShard(0, 0)
assert.Nil(t, replay)
require.NotNil(t, walk)
assert.Len(t, walk.actions, 1)
}
func TestRecoveryView_ActivatesEveryNonDisabledAction(t *testing.T) {
// Base snapshot has no PriorState → event-driven actions land inactive
// (pre-BootstrapComplete). Recovery view must clone them with active=
// true so the recovery walker emits matches for them.
rule := &s3lifecycle.Rule{
ID: "pending",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 7,
}
snap := New().Compile(
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
CompileOptions{}, // no PriorStates → action lands inactive
)
key := s3lifecycle.ActionKey{
Bucket: "b1",
RuleHash: s3lifecycle.RuleHash(rule),
ActionKind: s3lifecycle.ActionKindExpirationDays,
}
require.False(t, snap.actions[key].IsActive(), "precondition: base action must be inactive")
rv := RecoveryView(snap)
require.NotNil(t, rv)
clone := rv.actions[key]
require.NotNil(t, clone)
assert.True(t, clone.IsActive(), "recovery view must force-activate even pre-BootstrapComplete actions")
assert.Equal(t, ModeEventDriven, clone.Mode, "recovery preserves base Mode")
}
func TestRecoveryView_PreservesBaseModeForScanAtDate(t *testing.T) {
// ScanAtDate actions are active at compile time (no BootstrapComplete
// requirement). Recovery view must keep their Mode as ModeScanAtDate so
// the walker's date evaluator runs as expected — a Mode rewrite here
// would re-route them through the event-driven gate by mistake.
when := time.Now().Add(48 * time.Hour)
rule := &s3lifecycle.Rule{
ID: "d",
Status: s3lifecycle.StatusEnabled,
ExpirationDate: when,
}
snap := New().Compile(
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
CompileOptions{},
)
rv := RecoveryView(snap)
require.NotNil(t, rv)
require.Len(t, rv.actions, 1)
for _, a := range rv.actions {
assert.Equal(t, ModeScanAtDate, a.Mode)
assert.True(t, a.IsActive())
}
}
func TestRecoveryView_ExcludesDisabledRules(t *testing.T) {
rule := &s3lifecycle.Rule{
ID: "off",
Status: s3lifecycle.StatusDisabled,
ExpirationDays: 7,
}
snap := buildSnapshotForViews(t, "b1", rule)
rv := RecoveryView(snap)
assert.Nil(t, rv, "disabled rules must not surface in recovery view")
}
func TestRecoveryView_ClonesAreIndependentOfBase(t *testing.T) {
rule := &s3lifecycle.Rule{ID: "iso", Status: s3lifecycle.StatusEnabled, ExpirationDays: 3}
snap := buildSnapshotForViews(t, "b1", rule)
key := s3lifecycle.ActionKey{
Bucket: "b1",
RuleHash: s3lifecycle.RuleHash(rule),
ActionKind: s3lifecycle.ActionKindExpirationDays,
}
base := snap.Action(key)
baseActiveBefore := base.IsActive()
rv := RecoveryView(snap)
require.NotNil(t, rv)
clone := rv.actions[key]
require.NotNil(t, clone)
assert.NotSame(t, base, clone)
clone.engineState.Store(engineStateInactive)
assert.Equal(t, baseActiveBefore, base.IsActive(),
"flipping the recovery clone must not perturb the base")
}
func TestRecoveryView_NilSnapshot(t *testing.T) {
assert.Nil(t, RecoveryView(nil))
}