[s3] force filer apply s3 expiry metadata (#10469)

* fix: apply S3 Expiry Metadata

* add test Header X-Seaweedfs-Expires-S3

* resolve comments

* test entry lookup by mtime

* filer: skip s3 expiry stamp on versioned entries

The s3 expiry path skips entries carrying a version id, so stamping one
takes away its expiry rather than moving it onto mtime. Files under
.versions/ are written once, so crtime already tracks their needles.

---------

Co-authored-by: Konstantin Lebedev <whitefox@mayflower.work>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/filer/entry.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"os"
+	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -159,6 +160,51 @@ func (entry *Entry) IsExpireS3Enabled() (exist bool) {
 	return exist
 }
 
+func (entry *Entry) isS3Entry() bool {
+	if entry.Extended == nil {
+		return false
+	}
+
+	if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found {
+		return true
+	}
+
+	if _, found := entry.Extended[s3_constants.ExtETagKey]; found {
+		return true
+	}
+
+	for key := range entry.Extended {
+		if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func (entry *Entry) ApplyS3ExpiryMetadata() {
+	if entry.TtlSec == 0 {
+		return
+	}
+
+	if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found {
+		return
+	}
+
+	// The s3 expiry path skips versioned entries, so stamping one would drop its
+	// expiry entirely instead of moving it onto mtime.
+	if entry.IsS3Versioning() {
+		return
+	}
+
+	if !entry.isS3Entry() {
+		return
+	}
+
+	entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true")
+
+	return
+}
+
 func (entry *Entry) IsS3Versioning() (exist bool) {
 	if entry.Extended != nil {
 		_, exist = entry.Extended[s3_constants.ExtVersionIdKey]

weed/filer/entry_s3_expiry_test.go

@@ -0,0 +1,47 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func TestApplyS3ExpiryMetadata(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		ttlSec   int32
+		extended map[string][]byte
+		want     bool
+	}{
+		{
+			name:     "s3 entry with ttl",
+			ttlSec:   3600,
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
+			want:     true,
+		},
+		{
+			name:     "no ttl",
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
+		},
+		{
+			name:   "not an s3 entry",
+			ttlSec: 3600,
+		},
+		{
+			name:   "versioned entry keeps crtime expiry",
+			ttlSec: 3600,
+			extended: map[string][]byte{
+				s3_constants.ExtETagKey:      []byte("abc123"),
+				s3_constants.ExtVersionIdKey: []byte("v1"),
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			entry := &Entry{Attr: Attr{TtlSec: tc.ttlSec}, Extended: tc.extended}
+			entry.ApplyS3ExpiryMetadata()
+			if got := entry.IsExpireS3Enabled(); got != tc.want {
+				t.Fatalf("IsExpireS3Enabled() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

weed/s3api/s3_constants/extend_key.go

@@ -1,6 +1,7 @@
 package s3_constants
 
 const (
+	ExtAmzPrefix                = "Seaweed-X-Amz-"
 	ExtAmzOwnerKey              = "Seaweed-X-Amz-Owner"
 	ExtAmzAclKey                = "Seaweed-X-Amz-Acl"
 	ExtOwnershipKey             = "Seaweed-X-Amz-Ownership"

weed/server/filer_grpc_server.go

@@ -387,6 +387,7 @@ func (fs *FilerServer) applyStorageDefaultsToEntry(ctx context.Context, entry *f
 	} else if entry.TtlSec == 0 {
 		entry.TtlSec = so.TtlSeconds
 	}
+	entry.ApplyS3ExpiryMetadata()
 	return so, nil
 }
 

weed/server/filer_grpc_server_object_txn_ttl_test.go

@@ -36,12 +36,16 @@ func TestObjectTransactionPutAppliesConfiguredTTLAndExpires(t *testing.T) {
 				Entry: &filer_pb.Entry{
 					Name: "expired.mp4",
 					Attributes: &filer_pb.FuseAttributes{
-						Crtime:   old.Unix(),
+						Crtime:   time.Now().Unix(),
 						Mtime:    old.Unix(),
 						FileMode: 0644,
 						FileSize: 123,
 						TtlSec:   0,
 					},
+					Extended: map[string][]byte{
+						"Seaweed-X-Amz-Owner": []byte("admin"),
+						"Seaweed-X-Amz-ETag":  []byte("abc123"),
+					},
 				},
 			},
 		},
@@ -60,6 +64,9 @@ func TestObjectTransactionPutAppliesConfiguredTTLAndExpires(t *testing.T) {
 	if got := entry.TtlSec; got != 3600 {
 		t.Fatalf("TtlSec = %d, want 3600 from fs.configure storage rule", got)
 	}
+	if _, found := entry.Extended["X-Seaweedfs-Expires-S3"]; !found {
+		t.Fatalf("Header X-Seaweedfs-Expires-S3 not found")
+	}
 
 	_, err = fs.filer.FindEntry(context.Background(), util.FullPath("/buckets/video/expired.mp4"))
 	if err != filer_pb.ErrNotFound {
@@ -90,12 +97,16 @@ func TestObjectTransactionPutPreservesExplicitTTL(t *testing.T) {
 				Entry: &filer_pb.Entry{
 					Name: "lifecycle.mp4",
 					Attributes: &filer_pb.FuseAttributes{
-						Crtime:   now.Unix(),
+						Crtime:   time.Now().Add(-2 * time.Hour).Unix(),
 						Mtime:    now.Unix(),
 						FileMode: 0644,
 						FileSize: 123,
 						TtlSec:   7200,
 					},
+					Extended: map[string][]byte{
+						"Seaweed-X-Amz-Owner": []byte("admin"),
+						"Seaweed-X-Amz-ETag":  []byte("abc123"),
+					},
 				},
 			},
 		},
@@ -114,6 +125,9 @@ func TestObjectTransactionPutPreservesExplicitTTL(t *testing.T) {
 	if got := entry.TtlSec; got != 7200 {
 		t.Fatalf("TtlSec = %d, want explicit TTL 7200 to win over fs.configure", got)
 	}
+	if _, found := entry.Extended["X-Seaweedfs-Expires-S3"]; !found {
+		t.Fatalf("Header X-Seaweedfs-Expires-S3 not found")
+	}
 }
 
 func TestObjectTransactionPutClearsTTLForRemoteEntry(t *testing.T) {