Files
seaweedfs/weed/s3api/s3api_object_ownership_test.go
T
Chris LuandGitHub c4f0b12a9a s3: lazy, bounded, entry-free per-bucket caches (#10213)
* s3: stop warming every bucket's config at startup

Listing all buckets in BucketRegistry.init() made S3 gateway startup
O(buckets) and pinned every bucket's metadata and config resident, which
does not scale past a few hundred thousand buckets. Both caches already
have lazy miss paths, so load on first access instead and let the
metadata subscription refresh only entries already resident; cold
buckets cost one filer round-trip on their first request.

* s3: bound the per-bucket caches with LRU eviction

The bucket config cache, bucket registry, and their negative caches were
plain maps that only ever grew: the config cache TTL made Get miss but
never evicted the entry, and the not-found sets grew on every probe of a
nonexistent bucket name. Cap all four at 65536 entries with LRU eviction
so a gateway keeps its hot working set and evicted buckets reload from
the filer on next access.

* s3: cache parsed bucket config instead of the full filer entry

Each cached BucketConfig retained the whole bucket entry (extended
attribute map plus raw content bytes) alongside the fields parsed from
it, roughly doubling per-bucket cache cost and keeping data the read
path never looks at. Parse everything up front in
newBucketConfigFromEntry - now also the creator identity, tags,
encryption config, and stored lifecycle XML - and drop the entry.

updateBucketConfig now reads the entry fresh from the filer and diffs
the mapped extended attributes against it, so the patch is computed
against current state instead of a cached copy; the config clone
helpers that existed for that path go away.

* s3: dedup cold bucket-registry loads per bucket

The registry's notFound lock doubled as the load serializer, holding one
global mutex across the filer round-trip so first-touch requests for
different buckets queued behind each other; the cache fill also happened
after the lock was released, so two concurrent misses for the same
bucket could both reach the filer. Replace it with a singleflight per
bucket that fills the cache inside the flight: different buckets load
concurrently, the same bucket loads once.
2026-07-02 21:11:34 -07:00

194 lines
6.2 KiB
Go

package s3api
import (
"net/http"
"testing"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
func TestSetObjectOwnerFromRequest(t *testing.T) {
tests := []struct {
name string
bucketRegistryNil bool
bucketMetadata *BucketMetaData
bucketMetadataError s3err.ErrorCode
uploaderAccountId string
expectedOwnerId string
description string
}{
{
name: "BucketOwnerEnforced - use bucket owner",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
Owner: &s3.Owner{
ID: stringPtr("bucket-owner-123"),
DisplayName: stringPtr("Bucket Owner"),
},
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "uploader-456",
expectedOwnerId: "bucket-owner-123",
description: "Should use bucket owner when BucketOwnerEnforced",
},
{
name: "ObjectWriter - use uploader",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipObjectWriter,
Owner: &s3.Owner{
ID: stringPtr("bucket-owner-123"),
DisplayName: stringPtr("Bucket Owner"),
},
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should use uploader when ObjectWriter mode",
},
{
name: "BucketOwnerPreferred - use uploader",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipBucketOwnerPreferred,
Owner: &s3.Owner{
ID: stringPtr("bucket-owner-123"),
DisplayName: stringPtr("Bucket Owner"),
},
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should use uploader when BucketOwnerPreferred mode",
},
{
name: "BucketOwnerEnforced but owner is nil - fallback to uploader",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
Owner: nil,
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should fallback to uploader when bucket owner is nil",
},
{
name: "BucketOwnerEnforced but owner ID is nil - fallback to uploader",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
Owner: &s3.Owner{
ID: nil,
DisplayName: stringPtr("Bucket Owner"),
},
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should fallback to uploader when bucket owner ID is nil",
},
{
name: "Bucket metadata error - fallback to uploader",
bucketRegistryNil: false,
bucketMetadata: nil,
bucketMetadataError: s3err.ErrNoSuchBucket,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should fallback to uploader when bucket metadata unavailable",
},
{
name: "Bucket registry is nil - fallback to uploader",
bucketRegistryNil: true,
uploaderAccountId: "uploader-456",
expectedOwnerId: "uploader-456",
description: "Should fallback to uploader when bucketRegistry is nil",
},
{
name: "Empty uploader account ID - no owner set",
bucketRegistryNil: false,
bucketMetadata: &BucketMetaData{
Name: "test-bucket",
ObjectOwnership: s3_constants.OwnershipObjectWriter,
Owner: &s3.Owner{
ID: stringPtr("bucket-owner-123"),
DisplayName: stringPtr("Bucket Owner"),
},
},
bucketMetadataError: s3err.ErrNone,
uploaderAccountId: "",
expectedOwnerId: "",
description: "Should not set owner when uploader account ID is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create mock S3ApiServer
s3a := &S3ApiServer{}
// Setup bucket registry with mock behavior
if !tt.bucketRegistryNil {
// Create a minimal BucketRegistry with overridden GetBucketMetadata
s3a.bucketRegistry = NewBucketRegistry(nil)
// Pre-populate the cache with test metadata
if tt.bucketMetadata != nil && tt.bucketMetadataError == s3err.ErrNone {
s3a.bucketRegistry.metadataCache.Add("test-bucket", tt.bucketMetadata)
} else if tt.bucketMetadataError == s3err.ErrNoSuchBucket {
s3a.bucketRegistry.notFound.Add("test-bucket", struct{}{})
}
}
// Create mock request with uploader account ID
req, _ := http.NewRequest("PUT", "/test-bucket/test-object", nil)
req.Header.Set(s3_constants.AmzAccountId, tt.uploaderAccountId)
// Create entry
entry := &filer_pb.Entry{
Name: "test-object",
}
// Call the function
s3a.setObjectOwnerFromRequest(req, "test-bucket", entry)
// Verify the owner ID
if tt.expectedOwnerId == "" {
if entry.Extended != nil {
if _, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
t.Errorf("%s: Expected no owner to be set, but owner was set", tt.description)
}
}
} else {
if entry.Extended == nil {
t.Errorf("%s: Expected owner to be set, but Extended is nil", tt.description)
return
}
ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]
if !exists {
t.Errorf("%s: Expected owner to be set, but ExtAmzOwnerKey not found", tt.description)
return
}
actualOwnerId := string(ownerBytes)
if actualOwnerId != tt.expectedOwnerId {
t.Errorf("%s: Expected owner ID %s, got %s", tt.description, tt.expectedOwnerId, actualOwnerId)
}
}
})
}
}
// Helper function to create string pointers
func stringPtr(s string) *string {
return &s
}