diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 361e6828..05b7b4a7 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -47,6 +47,7 @@ import ( "github.com/google/uuid" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/debuglogger" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -110,11 +111,16 @@ type Azure struct { serviceURL string sasToken string copyObjectThreshold int64 + // copySASVersion overrides the service version used to sign the copy-source + // SAS for server-side copy. Empty means use the SDK default. This exists so + // endpoints that lag the SDK's SAS version (e.g. Azurite) can verify the + // signature; production leaves it unset. + copySASVersion string } var _ backend.Backend = &Azure{} -func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64) (*Azure, error) { +func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64, copySASVersion string) (*Azure, error) { url := serviceURL if serviceURL == "" && accountName != "" { // if not otherwise specified, use the typical form: @@ -132,6 +138,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho serviceURL: serviceURL, sasToken: sasToken, copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, }, nil } @@ -154,6 +161,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho serviceURL: url, defaultCreds: cred, copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, }, nil } @@ -172,6 +180,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho serviceURL: url, sharedkeyCreds: cred, copyObjectThreshold: copyObjectThreshold, + copySASVersion: copySASVersion, }, nil } @@ -1252,7 +1261,33 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu }, nil } - // Get the source object + // Cross-object copy: try server-side copy first, fall back to download+reupload. + srcClient, err := az.getBlobClient(srcBucket, srcObj) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + srcProps, err := srcClient.GetProperties(ctx, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + + if srcProps.ContentLength != nil && *srcProps.ContentLength > az.copyObjectThreshold { + return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold) + } + + out, err := az.serverSideCopyObject(ctx, input, srcBucket, srcObj, dstClient, srcClient, &srcProps) + if err == nil { + return out, nil + } + if !errors.Is(err, errServerSideCopyFallback) { + return s3response.CopyObjectOutput{}, err + } + + debuglogger.Logf("falling back to download+reupload (%q/%q -> %q/%q): %v", + srcBucket, srcObj, *input.Bucket, *input.Key, err) + + // Fallback: download and re-upload through the gateway. downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil) if err != nil { return s3response.CopyObjectOutput{}, azureErrToS3Err(err) diff --git a/backend/azure/copy.go b/backend/azure/copy.go new file mode 100644 index 00000000..7cd2a6c8 --- /dev/null +++ b/backend/azure/copy.go @@ -0,0 +1,304 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package azure + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +const copyPollInterval = 500 * time.Millisecond + +// errServerSideCopyFallback signals that server-side copy could not be started +// and the caller should fall back to download+reupload. Failures before the +// copy is started are wrapped in this sentinel; once StartCopyFromURL has +// succeeded the destination blob has been written, so later failures are +// returned as-is rather than silently re-copying through the gateway. +var errServerSideCopyFallback = errors.New("server-side copy unavailable") + +func (az *Azure) copySourceURL(ctx context.Context, srcBucket, srcObj string) (string, error) { + if az.sharedkeyCreds != nil { + now := time.Now().UTC() + sasQueryParams, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + Version: az.copySASVersion, + StartTime: now.Add(-10 * time.Second), + ExpiryTime: now.Add(15 * time.Minute), + Permissions: (&sas.BlobPermissions{Read: true}).String(), + ContainerName: srcBucket, + BlobName: srcObj, + }.SignWithSharedKey(az.sharedkeyCreds) + if err != nil { + return "", err + } + + return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil + } + + if az.sasToken != "" { + return az.getBlobURL(srcBucket, srcObj) + "?" + az.sasToken, nil + } + + if az.defaultCreds != nil { + svcClient, err := service.NewClient(az.serviceURL, az.defaultCreds, nil) + if err != nil { + return "", fmt.Errorf("init service client: %w", err) + } + + now := time.Now().UTC() + info := service.KeyInfo{ + Start: backend.GetPtrFromString(now.Add(-10 * time.Second).Format(sas.TimeFormat)), + Expiry: backend.GetPtrFromString(now.Add(48 * time.Hour).Format(sas.TimeFormat)), + } + + udc, err := svcClient.GetUserDelegationCredential(ctx, info, nil) + if err != nil { + return "", fmt.Errorf("get user delegation credential: %w", err) + } + + perms := &sas.BlobPermissions{Read: true} + sasQueryParams, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + Version: az.copySASVersion, + StartTime: now.Add(-10 * time.Second), + ExpiryTime: now.Add(15 * time.Minute), + Permissions: perms.String(), + ContainerName: srcBucket, + BlobName: srcObj, + }.SignWithUserDelegation(udc) + if err != nil { + return "", fmt.Errorf("sign user delegation sas: %w", err) + } + + return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil + } + + return "", errors.New("no credentials available") +} + +func (az *Azure) serverSideCopyObject( + ctx context.Context, + input s3response.CopyObjectInput, + srcBucket, srcObj string, + dstClient, srcClient *blob.Client, + srcProps *blob.GetPropertiesResponse, +) (s3response.CopyObjectOutput, error) { + srcURL, err := az.copySourceURL(ctx, srcBucket, srcObj) + if err != nil { + // Any failure to build a signed source URL means server-side copy can't + // be started at all, so fall back instead of failing the request. This + // notably covers GetUserDelegationCredential, which fails if the gateway + // identity lacks the Storage Blob Delegator role or the endpoint doesn't + // implement the delegation key API. + return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err) + } + + opts := &blob.StartCopyFromURLOptions{} + + // Copy Blob reads absent x-ms-meta-* headers as "inherit the source + // metadata", so an empty opts.Metadata cannot express "no metadata". When + // filtering empties the set, the destination metadata has to be cleared + // after the copy instead. + clearMetadata := false + + if input.MetadataDirective == types.MetadataDirectiveReplace { + meta := input.Metadata + if meta == nil { + meta = make(map[string]string) + } + if getString(input.Expires) != "" { + meta[string(keyExpires)] = *input.Expires + } + if getString(input.WebsiteRedirectLocation) != "" { + meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation + } + opts.Metadata = parseMetadata(meta) + } else { + // MetadataDirective COPY: StartCopyFromURL would otherwise copy the + // source blob's metadata verbatim, including the internal website-redirect + // key. Set the metadata explicitly so that key is dropped from the + // destination, matching the download+reupload fallback. + if srcProps == nil { + return s3response.CopyObjectOutput{}, fmt.Errorf( + "%w: source properties required to filter metadata", errServerSideCopyFallback) + } + if meta := parseAzMetadata(srcProps.Metadata); meta != nil { + delete(meta, string(keyWebsiteRedirect)) + opts.Metadata = parseMetadata(meta) + clearMetadata = len(meta) == 0 + } + } + + if input.TaggingDirective == types.TaggingDirectiveReplace { + tags, err := backend.ParseObjectTags(getString(input.Tagging)) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + opts.BlobTags = tags + } + + startResp, err := dstClient.StartCopyFromURL(ctx, srcURL, opts) + if err != nil { + return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err) + } + + finalProps, err := az.waitForCopy(ctx, dstClient, startResp.CopyStatus) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + if clearMetadata { + res, err := dstClient.SetMetadata(ctx, nil, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + if res.LastModified != nil { + finalProps.LastModified = res.LastModified + } + if res.ETag != nil { + finalProps.ETag = res.ETag + } + } + + if input.MetadataDirective == types.MetadataDirectiveReplace { + res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{ + BlobCacheControl: input.CacheControl, + BlobContentDisposition: input.ContentDisposition, + BlobContentEncoding: input.ContentEncoding, + BlobContentLanguage: input.ContentLanguage, + BlobContentType: input.ContentType, + }, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + if res.LastModified != nil { + finalProps.LastModified = res.LastModified + } + if res.ETag != nil { + finalProps.ETag = res.ETag + } + } + + if input.TaggingDirective == types.TaggingDirectiveCopy { + res, err := srcClient.GetTags(ctx, nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + _, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil) + if err != nil { + return s3response.CopyObjectOutput{}, azureErrToS3Err(err) + } + } + + if err := az.applyCopyObjectLock(ctx, *input.Bucket, *input.Key, input); err != nil { + return s3response.CopyObjectOutput{}, err + } + + var etag string + if finalProps.ETag != nil { + etag = convertAzureEtag(finalProps.ETag) + } else if startResp.ETag != nil { + etag = convertAzureEtag(startResp.ETag) + } + + lastModified := finalProps.LastModified + if lastModified == nil { + lastModified = startResp.LastModified + } + + return s3response.CopyObjectOutput{ + CopyObjectResult: &s3response.CopyObjectResult{ + LastModified: lastModified, + ETag: backend.GetPtrFromString(etag), + }, + }, nil +} + +func (az *Azure) waitForCopy(ctx context.Context, dstClient *blob.Client, initialStatus *blob.CopyStatusType) (*blob.GetPropertiesResponse, error) { + status := initialStatus + for status != nil && *status == blob.CopyStatusTypePending { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(copyPollInterval): + } + + props, err := dstClient.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + status = props.CopyStatus + } + + props, err := dstClient.GetProperties(ctx, nil) + if err != nil { + return nil, azureErrToS3Err(err) + } + + if props.CopyStatus != nil { + switch *props.CopyStatus { + case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted: + return nil, fmt.Errorf("blob copy failed with status %s", *props.CopyStatus) + } + } + + return &props, nil +} + +func (az *Azure) applyCopyObjectLock(ctx context.Context, bucket, key string, input s3response.CopyObjectInput) error { + if input.ObjectLockLegalHoldStatus != "" { + err := az.PutObjectLegalHold(ctx, bucket, key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return azureErrToS3Err(err) + } + } + + if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil { + retention := s3response.PutObjectRetentionInput{ + Mode: types.ObjectLockRetentionMode(input.ObjectLockMode), + RetainUntilDate: s3response.AmzDate{ + Time: *input.ObjectLockRetainUntilDate, + }, + } + + retParsed, err := json.Marshal(retention) + if err != nil { + return fmt.Errorf("parse object retention: %w", err) + } + err = az.PutObjectRetention(ctx, bucket, key, "", retParsed) + if err != nil { + if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) { + err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces) + } + return azureErrToS3Err(err) + } + } + + return nil +} diff --git a/backend/azure/copy_test.go b/backend/azure/copy_test.go new file mode 100644 index 00000000..a7ca16e5 --- /dev/null +++ b/backend/azure/copy_test.go @@ -0,0 +1,162 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package azure + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "net/url" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/versity/versitygw/s3response" +) + +const testServiceURL = "https://devstoreaccount1.blob.core.windows.net/devstoreaccount1" + +func testSharedKeyAzure(t *testing.T) *Azure { + t.Helper() + + // Any valid base64 key will do: the SAS is signed and inspected locally and + // never sent anywhere, so there is no need for a real account key. + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + t.Fatalf("generate account key: %v", err) + } + + cred, err := azblob.NewSharedKeyCredential("devstoreaccount1", + base64.StdEncoding.EncodeToString(raw)) + if err != nil { + t.Fatalf("NewSharedKeyCredential: %v", err) + } + + return &Azure{ + serviceURL: testServiceURL, + sharedkeyCreds: cred, + } +} + +// copySourceURL returns plain errors: the fallback classification is applied +// once, by its caller. Keeping it in a single place is what puts every failure +// to build a source URL on the fallback path, including the ones that only +// surface in production (GetUserDelegationCredential when the gateway identity +// lacks the Storage Blob Delegator role, or against an endpoint with no +// delegation key API). +func TestCopySourceURLErrorsAreNotPreClassified(t *testing.T) { + az := &Azure{serviceURL: testServiceURL} + + _, err := az.copySourceURL(context.Background(), "src-bucket", "src-object") + if err == nil { + t.Fatal("expected an error when no credentials are configured") + } + if errors.Is(err, errServerSideCopyFallback) { + t.Fatal("copySourceURL must not classify its own errors; its caller does") + } +} + +// The other half of that contract: an unsignable copy source has to reach the +// caller classified as "server-side copy unavailable", so that CopyObject falls +// back to download+reupload instead of failing the request. +func TestServerSideCopyObjectNoCredentialsFallsBack(t *testing.T) { + az := &Azure{serviceURL: testServiceURL} + + _, err := az.serverSideCopyObject(context.Background(), + s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil) + if err == nil { + t.Fatal("expected an error when no credentials are configured") + } + if !errors.Is(err, errServerSideCopyFallback) { + t.Fatalf("error must be classified as a fallback, got %v", err) + } +} + +// The MetadataDirective COPY path reads the source properties to filter the +// internal website-redirect key out of the destination metadata, so a caller +// that omits them must be turned away rather than panicking. Falling back keeps +// the copy correct, and skipping the filter instead would silently reintroduce +// the leaked redirect. +func TestServerSideCopyObjectNilSourcePropsFallsBack(t *testing.T) { + az := testSharedKeyAzure(t) + + _, err := az.serverSideCopyObject(context.Background(), + s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil) + if err == nil { + t.Fatal("expected an error when the source properties are missing") + } + if !errors.Is(err, errServerSideCopyFallback) { + t.Fatalf("error must be classified as a fallback, got %v", err) + } +} + +// The copy-source SAS has to be signed, read-only and scoped to the source +// blob. Its service version is pinned by configuration for endpoints that lag +// the SDK, and left at the SDK default otherwise; an override that silently +// stopped being applied would not fail the integration suite, because copies +// would fall back to download+reupload rather than error. +func TestCopySourceURLSharedKeySAS(t *testing.T) { + for _, tc := range []struct { + name string + version string + }{ + {name: "sdk default version", version: ""}, + {name: "pinned version", version: "2025-11-05"}, + } { + t.Run(tc.name, func(t *testing.T) { + az := testSharedKeyAzure(t) + az.copySASVersion = tc.version + + got, err := az.copySourceURL(context.Background(), "src-bucket", "src-object") + if err != nil { + t.Fatalf("copySourceURL: %v", err) + } + + base, query, ok := strings.Cut(got, "?") + if !ok { + t.Fatalf("expected a query string carrying the SAS, got %v", got) + } + if want := az.getBlobURL("src-bucket", "src-object"); base != want { + t.Fatalf("expected blob URL %v, got %v", want, base) + } + + vals, err := url.ParseQuery(query) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + if vals.Get("sig") == "" { + t.Error("expected a signature in the SAS") + } + if got := vals.Get("sp"); got != "r" { + t.Errorf("expected read-only permissions, got %q", got) + } + if got := vals.Get("sr"); got != "b" { + t.Errorf("expected a blob-scoped SAS, got %q", got) + } + + sv := vals.Get("sv") + if tc.version == "" { + if sv == "" { + t.Error("expected the SDK default service version to be filled in") + } + return + } + if sv != tc.version { + t.Errorf("expected service version %q, got %q", tc.version, sv) + } + }) + } +} diff --git a/cmd/internal/gwcli/azure.go b/cmd/internal/gwcli/azure.go index 37c8b696..5e1d3a6f 100644 --- a/cmd/internal/gwcli/azure.go +++ b/cmd/internal/gwcli/azure.go @@ -22,7 +22,7 @@ import ( ) var ( - azAccount, azKey, azServiceURL, azSASToken string + azAccount, azKey, azServiceURL, azSASToken, azCopySASVersion string ) // AzureCommand returns the "azure" subcommand, common to all versitygw @@ -62,12 +62,18 @@ func AzureCommand() *cli.Command { Aliases: []string{"u"}, Destination: &azServiceURL, }, + &cli.StringFlag{ + Name: "copy-sas-version", + Usage: "service version used to sign the copy-source SAS for server-side copy (defaults to the SDK version; set this for endpoints that lag the SDK, e.g. Azurite)", + EnvVars: []string{"AZ_COPY_SAS_VERSION"}, + Destination: &azCopySASVersion, + }, }, } } func runAzure(ctx *cli.Context) error { - be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold) + be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold, azCopySASVersion) if err != nil { return fmt.Errorf("init azure: %w", err) } diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index ffa45c3a..3d153ae1 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -28,6 +28,10 @@ services: - "10002:10002" restart: always hostname: azurite + # Server-side copy makes Azurite fetch the copy source from its own HTTPS + # endpoint; trust the self-signed test cert so that TLS validation succeeds. + environment: + NODE_EXTRA_CA_CERTS: /tests/certs/azurite.pem command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0 --skipApiVersionCheck" volumes: - ./tests/certs:/tests/certs @@ -39,4 +43,9 @@ services: - ./:/app ports: - 7070:7070 + # Azurite lags the SDK's SAS version and can't verify a copy-source SAS signed + # with it, so pin the copy-source SAS to a version Azurite supports. Production + # leaves this unset and uses the SDK default. + environment: + AZ_COPY_SAS_VERSION: "2025-11-05" command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"] diff --git a/tests/integration/CopyObject.go b/tests/integration/CopyObject.go index 63d09bac..0e5a65af 100644 --- a/tests/integration/CopyObject.go +++ b/tests/integration/CopyObject.go @@ -729,6 +729,57 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error { }) } +func CopyObject_should_not_copy_website_redirect_without_user_metadata(s *S3Conf) error { + testName := "CopyObject_should_not_copy_website_redirect_without_user_metadata" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj, dstObj := "source-object", "dest-object" + redirectLocation := "/source-redirect" + + // The redirect location is the only thing set on the source: no user + // metadata and no Expires. Backends that keep it alongside user metadata + // have to drop it on copy, and the filtered set is empty here, so a + // backend whose copy treats "empty metadata" the same as "metadata not + // specified" will silently inherit the source's redirect instead. + _, err := putObjectWithData(int64(100), &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + WebsiteRedirectLocation: &redirectLocation, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &dstObj, + CopySource: getPtr(bucket + "/" + srcObj), + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &dstObj, + }) + cancel() + if err != nil { + return err + } + if got := getString(out.WebsiteRedirectLocation); got != "" { + return fmt.Errorf("expected WebsiteRedirectLocation not to be copied, got %v", got) + } + if len(out.Metadata) != 0 { + return fmt.Errorf("expected no user metadata on the destination, instead got %v", out.Metadata) + } + + return nil + }) +} + func CopyObject_should_replace_meta_props(s *S3Conf) error { testName := "CopyObject_should_replace_meta_props" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -1622,6 +1673,105 @@ func CopyObject_success(s *S3Conf) error { }) } +// CopyObject_cross_bucket_server_side_copy exercises a cross-bucket copy where +// source and destination live in different buckets. On the Azure backend this +// drives the server-side StartCopyFromURL path (added in copy.go), verifying that +// object data, user metadata, content-type and tags all survive the copy and that +// an ETag is returned. It also runs on the other backends as a plain copy. +func CopyObject_cross_bucket_server_side_copy(s *S3Conf) error { + testName := "CopyObject_cross_bucket_server_side_copy" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + srcObj, dstObj := "source-object", "dest-object" + dstBucket := getBucketName() + if err := setup(s, dstBucket); err != nil { + return err + } + + dataLength := int64(1234567) + cType := "application/json" + meta := map[string]string{ + "foo": "bar", + "baz": "quxx", + } + tagging := "foo=bar&baz=quxx" + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &srcObj, + ContentType: &cType, + Metadata: meta, + Tagging: &tagging, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)), + }) + cancel() + if err != nil { + return err + } + if copyOut.CopyObjectResult == nil || getString(copyOut.CopyObjectResult.ETag) == "" { + return fmt.Errorf("expected non-empty ETag in copy result") + } + + // Object data must be byte-for-byte identical. + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &dstBucket, + Key: &dstObj, + }) + if err != nil { + cancel() + return err + } + bdy, err := io.ReadAll(out.Body) + out.Body.Close() + cancel() + if err != nil { + return err + } + if sha256.Sum256(bdy) != r.csum { + return fmt.Errorf("invalid object data after copy") + } + + // Content-type and user metadata must be preserved (MetadataDirective COPY). + if err := checkObjectMetaProps(s3client, dstBucket, dstObj, ObjectMetaProps{ + ContentLength: dataLength, + ContentType: cType, + Metadata: meta, + }); err != nil { + return err + } + + // Tags must be preserved (TaggingDirective COPY). + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + tagRes, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &dstBucket, + Key: &dstObj, + }) + cancel() + if err != nil { + return err + } + expectedTagSet := []types.Tag{ + {Key: getPtr("foo"), Value: getPtr("bar")}, + {Key: getPtr("baz"), Value: getPtr("quxx")}, + } + if !areTagsSame(tagRes.TagSet, expectedTagSet) { + return fmt.Errorf("expected the tag set to be %v, instead got %v", + expectedTagSet, tagRes.TagSet) + } + + return teardown(s, dstBucket) + }) +} + func CopyObject_with_special_characters(s *S3Conf) error { testName := "CopyObject_with_special_characters" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 401a5ccd..2e6cd4de 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -370,6 +370,7 @@ func TestCopyObject(ts *TestState) { ts.Run(CopyObject_invalid_copy_source) ts.Run(CopyObject_non_existing_dir_object) ts.Run(CopyObject_should_copy_meta_props) + ts.Run(CopyObject_should_not_copy_website_redirect_without_user_metadata) ts.Run(CopyObject_should_replace_meta_props) ts.Run(CopyObject_invalid_website_redirect_location) ts.Run(CopyObject_default_content_type_with_replace_metadata) @@ -392,6 +393,7 @@ func TestCopyObject(ts *TestState) { } ts.Run(CopyObject_with_special_characters) ts.Run(CopyObject_success) + ts.Run(CopyObject_cross_bucket_server_side_copy) ts.Run(CopyObject_incorrect_source_bucket_expected_owner) } @@ -2656,6 +2658,7 @@ func GetIntTests() IntTests { "CopyObject_invalid_copy_source": CopyObject_invalid_copy_source, "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, "CopyObject_should_copy_meta_props": CopyObject_should_copy_meta_props, + "CopyObject_should_not_copy_website_redirect_without_user_metadata": CopyObject_should_not_copy_website_redirect_without_user_metadata, "CopyObject_should_replace_meta_props": CopyObject_should_replace_meta_props, "CopyObject_invalid_website_redirect_location": CopyObject_invalid_website_redirect_location, "CopyObject_default_content_type_with_replace_metadata": CopyObject_default_content_type_with_replace_metadata, @@ -2674,6 +2677,7 @@ func GetIntTests() IntTests { "CopyObject_to_itself_by_replacing_the_checksum": CopyObject_to_itself_by_replacing_the_checksum, "CopyObject_with_special_characters": CopyObject_with_special_characters, "CopyObject_success": CopyObject_success, + "CopyObject_cross_bucket_server_side_copy": CopyObject_cross_bucket_server_side_copy, "CopyObject_incorrect_source_bucket_expected_owner": CopyObject_incorrect_source_bucket_expected_owner, "PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object, "PutObjectTagging_long_tags": PutObjectTagging_long_tags,