Compare commits

...
Author SHA1 Message Date
Chris Lu 7f1bded535 fix(s3api): reject lifecycle rules whose TTL overflows int32 seconds
bucketLifecycleTTLRule.TtlSec is int32. AWS allows Expiration.Days up
to 2,147,483,647 which would silently overflow when multiplied by
LifeCycleInterval. Compute in int64, return InvalidRequest above
math.MaxInt32 seconds.
2026-05-07 22:28:20 -07:00
Chris Lu aaaf2fb53f refactor(s3api): drop redundant prefix normalization on read path
encodeBucketLifecycleTTLRules already normalizes rule.Prefix before it
hits the filer; matchBucketLifecycleTTLSeconds is on the per-object
hot path and shouldn't re-normalize.
2026-05-07 22:28:20 -07:00
Chris Lu 34922fd761 s3api: persist lifecycle TTL rules and enforce on assign 2026-02-11 09:45:50 -08:00
5 changed files with 227 additions and 3 deletions
+3 -2
View File
@@ -11,7 +11,7 @@ const (
ExtETagKey = "Seaweed-X-Amz-ETag"
ExtLatestVersionIdKey = "Seaweed-X-Amz-Latest-Version-Id"
ExtLatestVersionFileNameKey = "Seaweed-X-Amz-Latest-Version-File-Name"
ExtAllowEmptyFolders = "Seaweed-X-Amz-Allow-Empty-Folders"
ExtAllowEmptyFolders = "Seaweed-X-Amz-Allow-Empty-Folders"
// Cached list metadata in .versions directory for single-scan efficiency
ExtLatestVersionSizeKey = "Seaweed-X-Amz-Latest-Version-Size"
ExtLatestVersionETagKey = "Seaweed-X-Amz-Latest-Version-ETag"
@@ -21,7 +21,8 @@ const (
ExtMultipartObjectKey = "key"
// Bucket Policy
ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
ExtBucketLifecycleTTLRulesKey = "Seaweed-X-Amz-Bucket-Lifecycle-TTL-Rules"
// Object Retention and Legal Hold
ExtObjectLockModeKey = "Seaweed-X-Amz-Object-Lock-Mode"
+26 -1
View File
@@ -860,6 +860,7 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr
collectionName := s3a.getCollectionName(bucket)
collectionTtls := fc.GetCollectionTtls(collectionName)
changed := false
lifecycleRules := make([]bucketLifecycleTTLRule, 0, len(lifeCycleConfig.Rules))
for _, rule := range lifeCycleConfig.Rules {
if rule.Status != Enabled {
@@ -879,6 +880,21 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr
if rule.Expiration.Days == 0 {
continue
}
// Compute in int64 first, then refuse rules whose TTL won't fit
// int32 seconds. AWS allows Days up to 2,147,483,647, which would
// silently overflow if we cast directly.
ttlSecondsInt64 := int64(rule.Expiration.Days) * int64(util.LifeCycleInterval/time.Second)
if ttlSecondsInt64 > math.MaxInt32 {
glog.Errorf("PutBucketLifecycleConfigurationHandler %s: Expiration.Days=%d exceeds int32-second TTL", bucket, rule.Expiration.Days)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
ttlSec := int32(ttlSecondsInt64)
lifecycleRules = append(lifecycleRules, bucketLifecycleTTLRule{
Prefix: rulePrefix,
TtlSec: ttlSec,
})
locationPrefix := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix)
locConf := &filer_pb.FilerConf_PathConf{
LocationPrefix: locationPrefix,
@@ -893,7 +909,6 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
ttlSec := int32((time.Duration(rule.Expiration.Days) * util.LifeCycleInterval).Seconds())
glog.V(2).Infof("Start updating TTL for %s", locationPrefix)
if updErr := s3a.updateEntriesTTL(locationPrefix, ttlSec); updErr != nil {
glog.Errorf("PutBucketLifecycleConfigurationHandler update TTL for %s: %s", locationPrefix, updErr)
@@ -917,6 +932,11 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr
return
}
}
if err := s3a.persistBucketLifecycleTTLRules(bucket, lifecycleRules); err != nil {
glog.Errorf("PutBucketLifecycleConfigurationHandler persist lifecycle metadata for %s: %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
writeSuccessResponseEmpty(w, r)
}
@@ -967,6 +987,11 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h
return
}
}
if err := s3a.persistBucketLifecycleTTLRules(bucket, nil); err != nil {
glog.Errorf("DeleteBucketLifecycleHandler clear lifecycle metadata for %s: %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
}
+125
View File
@@ -0,0 +1,125 @@
package s3api
import (
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
type bucketLifecycleTTLRule struct {
Prefix string `json:"prefix"`
TtlSec int32 `json:"ttlSec"`
}
func normalizeLifecycleRulePrefix(prefix string) string {
prefix = strings.TrimSpace(prefix)
prefix = strings.TrimPrefix(prefix, "/")
if prefix == "." || prefix == "/" {
return ""
}
return prefix
}
func encodeBucketLifecycleTTLRules(rules []bucketLifecycleTTLRule) ([]byte, error) {
if len(rules) == 0 {
return nil, nil
}
sorted := make([]bucketLifecycleTTLRule, 0, len(rules))
for _, rule := range rules {
if rule.TtlSec <= 0 {
continue
}
sorted = append(sorted, bucketLifecycleTTLRule{
Prefix: normalizeLifecycleRulePrefix(rule.Prefix),
TtlSec: rule.TtlSec,
})
}
if len(sorted) == 0 {
return nil, nil
}
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Prefix < sorted[j].Prefix
})
return json.Marshal(sorted)
}
func decodeBucketLifecycleTTLRules(extended map[string][]byte) []bucketLifecycleTTLRule {
if len(extended) == 0 {
return nil
}
serialized := extended[s3_constants.ExtBucketLifecycleTTLRulesKey]
if len(serialized) == 0 {
return nil
}
var rules []bucketLifecycleTTLRule
if err := json.Unmarshal(serialized, &rules); err != nil {
glog.Warningf("decode bucket lifecycle ttl rules: %v", err)
return nil
}
return rules
}
func matchBucketLifecycleTTLSeconds(rules []bucketLifecycleTTLRule, objectKey string) int32 {
if len(rules) == 0 {
return 0
}
objectKey = normalizeLifecycleRulePrefix(objectKey)
var bestPrefix string
var ttlSeconds int32
for _, rule := range rules {
// rule.Prefix is already normalized at encode time.
prefix := rule.Prefix
if prefix != "" && !strings.HasPrefix(objectKey, prefix) {
continue
}
if len(prefix) >= len(bestPrefix) {
bestPrefix = prefix
ttlSeconds = rule.TtlSec
}
}
return ttlSeconds
}
func (s3a *S3ApiServer) resolveBucketLifecycleTTLSeconds(bucket, filePath string) int32 {
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone || config == nil || config.Entry == nil {
return 0
}
bucketPrefix := s3a.bucketDir(bucket) + "/"
objectKey := strings.TrimPrefix(filePath, bucketPrefix)
if objectKey == filePath {
return 0
}
rules := decodeBucketLifecycleTTLRules(config.Entry.Extended)
return matchBucketLifecycleTTLSeconds(rules, objectKey)
}
func (s3a *S3ApiServer) persistBucketLifecycleTTLRules(bucket string, rules []bucketLifecycleTTLRule) error {
serialized, err := encodeBucketLifecycleTTLRules(rules)
if err != nil {
return fmt.Errorf("encode lifecycle ttl rules: %w", err)
}
errCode := s3a.updateBucketConfig(bucket, func(config *BucketConfig) error {
if config.Entry == nil {
return fmt.Errorf("bucket %s has no entry", bucket)
}
if config.Entry.Extended == nil {
config.Entry.Extended = make(map[string][]byte)
}
if len(serialized) == 0 {
delete(config.Entry.Extended, s3_constants.ExtBucketLifecycleTTLRulesKey)
} else {
config.Entry.Extended[s3_constants.ExtBucketLifecycleTTLRulesKey] = serialized
}
return nil
})
if errCode != s3err.ErrNone {
return fmt.Errorf("persist lifecycle ttl rules: %v", errCode)
}
return nil
}
@@ -0,0 +1,68 @@
package s3api
import (
"encoding/json"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
func TestMatchBucketLifecycleTTLSeconds(t *testing.T) {
rules := []bucketLifecycleTTLRule{
{Prefix: "", TtlSec: 3600},
{Prefix: "logs/", TtlSec: 7200},
{Prefix: "logs/archive/", TtlSec: 10800},
}
if got := matchBucketLifecycleTTLSeconds(rules, "logs/file.txt"); got != 7200 {
t.Fatalf("expected 7200 for logs/file.txt, got %d", got)
}
if got := matchBucketLifecycleTTLSeconds(rules, "logs/archive/file.txt"); got != 10800 {
t.Fatalf("expected 10800 for logs/archive/file.txt, got %d", got)
}
if got := matchBucketLifecycleTTLSeconds(rules, "other/file.txt"); got != 3600 {
t.Fatalf("expected 3600 for other/file.txt, got %d", got)
}
}
func TestDecodeBucketLifecycleTTLRules(t *testing.T) {
extended := map[string][]byte{
s3_constants.ExtBucketLifecycleTTLRulesKey: []byte(`[{"prefix":"logs/","ttlSec":86400}]`),
}
rules := decodeBucketLifecycleTTLRules(extended)
if len(rules) != 1 {
t.Fatalf("expected 1 rule, got %d", len(rules))
}
if rules[0].Prefix != "logs/" || rules[0].TtlSec != 86400 {
t.Fatalf("unexpected rule: %+v", rules[0])
}
extended[s3_constants.ExtBucketLifecycleTTLRulesKey] = []byte(`{invalid json`)
if got := decodeBucketLifecycleTTLRules(extended); got != nil {
t.Fatalf("expected nil rules for invalid JSON, got %+v", got)
}
}
func TestEncodeBucketLifecycleTTLRules(t *testing.T) {
serialized, err := encodeBucketLifecycleTTLRules([]bucketLifecycleTTLRule{
{Prefix: "/logs/", TtlSec: 86400},
{Prefix: "tmp/", TtlSec: 0},
{Prefix: "", TtlSec: 3600},
})
if err != nil {
t.Fatalf("encode failed: %v", err)
}
var decoded []bucketLifecycleTTLRule
if err := json.Unmarshal(serialized, &decoded); err != nil {
t.Fatalf("unmarshal failed: %v", err)
}
if len(decoded) != 2 {
t.Fatalf("expected 2 persisted rules, got %d", len(decoded))
}
if decoded[0].Prefix != "" || decoded[0].TtlSec != 3600 {
t.Fatalf("unexpected first rule: %+v", decoded[0])
}
if decoded[1].Prefix != "logs/" || decoded[1].TtlSec != 86400 {
t.Fatalf("unexpected second rule: %+v", decoded[1])
}
}
+5
View File
@@ -358,6 +358,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
if s3a.option.FilerGroup != "" {
collection = s3a.getCollectionName(bucket)
}
ttlSeconds := s3a.resolveBucketLifecycleTTLSeconds(bucket, filePath)
// Create assign function for chunked upload
assignFunc := func(ctx context.Context, count int) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
@@ -370,6 +371,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
DiskType: "",
DataCenter: s3a.option.DataCenter,
Path: filePath,
TtlSec: ttlSeconds,
})
if err != nil {
return fmt.Errorf("assign volume: %w", err)
@@ -534,6 +536,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
Chunks: chunkResult.FileChunks, // All chunks from auto-chunking
Extended: make(map[string][]byte),
}
if ttlSeconds > 0 {
entry.Attributes.TtlSec = ttlSeconds
}
// Always set Md5 attribute for regular object uploads (PutObject)
// This ensures the ETag is a pure MD5 hash, which AWS S3 SDKs expect