From d1f503181b29b8616f83783e4d0f591e7743a747 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:55:20 +0500 Subject: [PATCH] [s3] force filer apply s3 expiry metadata (#10469) * fix: apply S3 Expiry Metadata * add test Header X-Seaweedfs-Expires-S3 * resolve comments * test entry lookup by mtime * filer: skip s3 expiry stamp on versioned entries The s3 expiry path skips entries carrying a version id, so stamping one takes away its expiry rather than moving it onto mtime. Files under .versions/ are written once, so crtime already tracks their needles. --------- Co-authored-by: Konstantin Lebedev Co-authored-by: Chris Lu --- weed/filer/entry.go | 46 ++++++++++++++++++ weed/filer/entry_s3_expiry_test.go | 47 +++++++++++++++++++ weed/s3api/s3_constants/extend_key.go | 1 + weed/server/filer_grpc_server.go | 1 + .../filer_grpc_server_object_txn_ttl_test.go | 18 ++++++- 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 weed/filer/entry_s3_expiry_test.go diff --git a/weed/filer/entry.go b/weed/filer/entry.go index 205d7b79c..8a80c7462 100644 --- a/weed/filer/entry.go +++ b/weed/filer/entry.go @@ -2,6 +2,7 @@ package filer import ( "os" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -159,6 +160,51 @@ func (entry *Entry) IsExpireS3Enabled() (exist bool) { return exist } +func (entry *Entry) isS3Entry() bool { + if entry.Extended == nil { + return false + } + + if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found { + return true + } + + if _, found := entry.Extended[s3_constants.ExtETagKey]; found { + return true + } + + for key := range entry.Extended { + if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) { + return true + } + } + return false +} + +func (entry *Entry) ApplyS3ExpiryMetadata() { + if entry.TtlSec == 0 { + return + } + + if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found { + return + } + + // The s3 expiry path skips versioned entries, so stamping one would drop its + // expiry entirely instead of moving it onto mtime. + if entry.IsS3Versioning() { + return + } + + if !entry.isS3Entry() { + return + } + + entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true") + + return +} + func (entry *Entry) IsS3Versioning() (exist bool) { if entry.Extended != nil { _, exist = entry.Extended[s3_constants.ExtVersionIdKey] diff --git a/weed/filer/entry_s3_expiry_test.go b/weed/filer/entry_s3_expiry_test.go new file mode 100644 index 000000000..a968e74ae --- /dev/null +++ b/weed/filer/entry_s3_expiry_test.go @@ -0,0 +1,47 @@ +package filer + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +func TestApplyS3ExpiryMetadata(t *testing.T) { + for _, tc := range []struct { + name string + ttlSec int32 + extended map[string][]byte + want bool + }{ + { + name: "s3 entry with ttl", + ttlSec: 3600, + extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")}, + want: true, + }, + { + name: "no ttl", + extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")}, + }, + { + name: "not an s3 entry", + ttlSec: 3600, + }, + { + name: "versioned entry keeps crtime expiry", + ttlSec: 3600, + extended: map[string][]byte{ + s3_constants.ExtETagKey: []byte("abc123"), + s3_constants.ExtVersionIdKey: []byte("v1"), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + entry := &Entry{Attr: Attr{TtlSec: tc.ttlSec}, Extended: tc.extended} + entry.ApplyS3ExpiryMetadata() + if got := entry.IsExpireS3Enabled(); got != tc.want { + t.Fatalf("IsExpireS3Enabled() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/weed/s3api/s3_constants/extend_key.go b/weed/s3api/s3_constants/extend_key.go index c6c28b0d9..fe7c9ec81 100644 --- a/weed/s3api/s3_constants/extend_key.go +++ b/weed/s3api/s3_constants/extend_key.go @@ -1,6 +1,7 @@ package s3_constants const ( + ExtAmzPrefix = "Seaweed-X-Amz-" ExtAmzOwnerKey = "Seaweed-X-Amz-Owner" ExtAmzAclKey = "Seaweed-X-Amz-Acl" ExtOwnershipKey = "Seaweed-X-Amz-Ownership" diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 02d857e32..ed15fabcc 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -387,6 +387,7 @@ func (fs *FilerServer) applyStorageDefaultsToEntry(ctx context.Context, entry *f } else if entry.TtlSec == 0 { entry.TtlSec = so.TtlSeconds } + entry.ApplyS3ExpiryMetadata() return so, nil } diff --git a/weed/server/filer_grpc_server_object_txn_ttl_test.go b/weed/server/filer_grpc_server_object_txn_ttl_test.go index 124505036..70c355e0a 100644 --- a/weed/server/filer_grpc_server_object_txn_ttl_test.go +++ b/weed/server/filer_grpc_server_object_txn_ttl_test.go @@ -36,12 +36,16 @@ func TestObjectTransactionPutAppliesConfiguredTTLAndExpires(t *testing.T) { Entry: &filer_pb.Entry{ Name: "expired.mp4", Attributes: &filer_pb.FuseAttributes{ - Crtime: old.Unix(), + Crtime: time.Now().Unix(), Mtime: old.Unix(), FileMode: 0644, FileSize: 123, TtlSec: 0, }, + Extended: map[string][]byte{ + "Seaweed-X-Amz-Owner": []byte("admin"), + "Seaweed-X-Amz-ETag": []byte("abc123"), + }, }, }, }, @@ -60,6 +64,9 @@ func TestObjectTransactionPutAppliesConfiguredTTLAndExpires(t *testing.T) { if got := entry.TtlSec; got != 3600 { t.Fatalf("TtlSec = %d, want 3600 from fs.configure storage rule", got) } + if _, found := entry.Extended["X-Seaweedfs-Expires-S3"]; !found { + t.Fatalf("Header X-Seaweedfs-Expires-S3 not found") + } _, err = fs.filer.FindEntry(context.Background(), util.FullPath("/buckets/video/expired.mp4")) if err != filer_pb.ErrNotFound { @@ -90,12 +97,16 @@ func TestObjectTransactionPutPreservesExplicitTTL(t *testing.T) { Entry: &filer_pb.Entry{ Name: "lifecycle.mp4", Attributes: &filer_pb.FuseAttributes{ - Crtime: now.Unix(), + Crtime: time.Now().Add(-2 * time.Hour).Unix(), Mtime: now.Unix(), FileMode: 0644, FileSize: 123, TtlSec: 7200, }, + Extended: map[string][]byte{ + "Seaweed-X-Amz-Owner": []byte("admin"), + "Seaweed-X-Amz-ETag": []byte("abc123"), + }, }, }, }, @@ -114,6 +125,9 @@ func TestObjectTransactionPutPreservesExplicitTTL(t *testing.T) { if got := entry.TtlSec; got != 7200 { t.Fatalf("TtlSec = %d, want explicit TTL 7200 to win over fs.configure", got) } + if _, found := entry.Extended["X-Seaweedfs-Expires-S3"]; !found { + t.Fatalf("Header X-Seaweedfs-Expires-S3 not found") + } } func TestObjectTransactionPutClearsTTLForRemoteEntry(t *testing.T) {