feat(azure): server-side copy with download+reupload fallback

* feat(azure): implement server-side copy with fallback

Add server-side object copy for the Azure backend using StartCopyFromURL, with a
fallback to download+reupload when server-side copy is unavailable. The copy
logic lives in backend/azure/copy.go and handles metadata, tagging and object
lock configurations.

The copy-source SAS service version is configurable via the --copy-sas-version
flag (AZ_COPY_SAS_VERSION) and defaults to the SDK version, so production is
unchanged. Endpoints that lag the SDK's SAS version (e.g. Azurite) cannot verify
a SAS signed with the newer version and can set an older one. On a metadata-COPY,
the internal website-redirect key is dropped from the destination to match the
download+reupload fallback.

Testing:
- Add CopyObject_cross_bucket_server_side_copy, which copies an object with data,
  user metadata, content-type and tags across two buckets and verifies all are
  preserved and an ETag is returned.
- Configure the Azurite functional-test gateway with AZ_COPY_SAS_VERSION and let
  Azurite trust its self-signed test certificate (NODE_EXTRA_CA_CERTS) so it can
  fetch the copy source from its own HTTPS endpoint, ensuring CI exercises the
  real server-side copy path instead of always falling back.

Signed-off-by: Nils Leger <nils.leger@getflip.com>

* docs: update copyright year

Signed-off-by: Nils Leger <nils.leger@getflip.com>

* fix: always fallback to download+upload whenever there is an error building the server-side copy URL

Signed-off-by: Nils Leger <nils.leger@getflip.com>

---------

Signed-off-by: Nils Leger <nils.leger@getflip.com>
This commit is contained in:
Nils Leger
2026-08-27 16:37:28 -07:00
committed by GitHub
parent 29bf076abc
commit a780f3473a
7 changed files with 674 additions and 4 deletions
+37 -2
View File
@@ -47,6 +47,7 @@ import (
"github.com/google/uuid"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
@@ -110,11 +111,16 @@ type Azure struct {
serviceURL string
sasToken string
copyObjectThreshold int64
// copySASVersion overrides the service version used to sign the copy-source
// SAS for server-side copy. Empty means use the SDK default. This exists so
// endpoints that lag the SDK's SAS version (e.g. Azurite) can verify the
// signature; production leaves it unset.
copySASVersion string
}
var _ backend.Backend = &Azure{}
func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64) (*Azure, error) {
func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64, copySASVersion string) (*Azure, error) {
url := serviceURL
if serviceURL == "" && accountName != "" {
// if not otherwise specified, use the typical form:
@@ -132,6 +138,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
serviceURL: serviceURL,
sasToken: sasToken,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
@@ -154,6 +161,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
serviceURL: url,
defaultCreds: cred,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
@@ -172,6 +180,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
serviceURL: url,
sharedkeyCreds: cred,
copyObjectThreshold: copyObjectThreshold,
copySASVersion: copySASVersion,
}, nil
}
@@ -1252,7 +1261,33 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
}, nil
}
// Get the source object
// Cross-object copy: try server-side copy first, fall back to download+reupload.
srcClient, err := az.getBlobClient(srcBucket, srcObj)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcProps, err := srcClient.GetProperties(ctx, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
if srcProps.ContentLength != nil && *srcProps.ContentLength > az.copyObjectThreshold {
return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold)
}
out, err := az.serverSideCopyObject(ctx, input, srcBucket, srcObj, dstClient, srcClient, &srcProps)
if err == nil {
return out, nil
}
if !errors.Is(err, errServerSideCopyFallback) {
return s3response.CopyObjectOutput{}, err
}
debuglogger.Logf("falling back to download+reupload (%q/%q -> %q/%q): %v",
srcBucket, srcObj, *input.Bucket, *input.Key, err)
// Fallback: download and re-upload through the gateway.
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
+304
View File
@@ -0,0 +1,304 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package azure
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
const copyPollInterval = 500 * time.Millisecond
// errServerSideCopyFallback signals that server-side copy could not be started
// and the caller should fall back to download+reupload. Failures before the
// copy is started are wrapped in this sentinel; once StartCopyFromURL has
// succeeded the destination blob has been written, so later failures are
// returned as-is rather than silently re-copying through the gateway.
var errServerSideCopyFallback = errors.New("server-side copy unavailable")
func (az *Azure) copySourceURL(ctx context.Context, srcBucket, srcObj string) (string, error) {
if az.sharedkeyCreds != nil {
now := time.Now().UTC()
sasQueryParams, err := sas.BlobSignatureValues{
Protocol: sas.ProtocolHTTPS,
Version: az.copySASVersion,
StartTime: now.Add(-10 * time.Second),
ExpiryTime: now.Add(15 * time.Minute),
Permissions: (&sas.BlobPermissions{Read: true}).String(),
ContainerName: srcBucket,
BlobName: srcObj,
}.SignWithSharedKey(az.sharedkeyCreds)
if err != nil {
return "", err
}
return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil
}
if az.sasToken != "" {
return az.getBlobURL(srcBucket, srcObj) + "?" + az.sasToken, nil
}
if az.defaultCreds != nil {
svcClient, err := service.NewClient(az.serviceURL, az.defaultCreds, nil)
if err != nil {
return "", fmt.Errorf("init service client: %w", err)
}
now := time.Now().UTC()
info := service.KeyInfo{
Start: backend.GetPtrFromString(now.Add(-10 * time.Second).Format(sas.TimeFormat)),
Expiry: backend.GetPtrFromString(now.Add(48 * time.Hour).Format(sas.TimeFormat)),
}
udc, err := svcClient.GetUserDelegationCredential(ctx, info, nil)
if err != nil {
return "", fmt.Errorf("get user delegation credential: %w", err)
}
perms := &sas.BlobPermissions{Read: true}
sasQueryParams, err := sas.BlobSignatureValues{
Protocol: sas.ProtocolHTTPS,
Version: az.copySASVersion,
StartTime: now.Add(-10 * time.Second),
ExpiryTime: now.Add(15 * time.Minute),
Permissions: perms.String(),
ContainerName: srcBucket,
BlobName: srcObj,
}.SignWithUserDelegation(udc)
if err != nil {
return "", fmt.Errorf("sign user delegation sas: %w", err)
}
return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil
}
return "", errors.New("no credentials available")
}
func (az *Azure) serverSideCopyObject(
ctx context.Context,
input s3response.CopyObjectInput,
srcBucket, srcObj string,
dstClient, srcClient *blob.Client,
srcProps *blob.GetPropertiesResponse,
) (s3response.CopyObjectOutput, error) {
srcURL, err := az.copySourceURL(ctx, srcBucket, srcObj)
if err != nil {
// Any failure to build a signed source URL means server-side copy can't
// be started at all, so fall back instead of failing the request. This
// notably covers GetUserDelegationCredential, which fails if the gateway
// identity lacks the Storage Blob Delegator role or the endpoint doesn't
// implement the delegation key API.
return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err)
}
opts := &blob.StartCopyFromURLOptions{}
// Copy Blob reads absent x-ms-meta-* headers as "inherit the source
// metadata", so an empty opts.Metadata cannot express "no metadata". When
// filtering empties the set, the destination metadata has to be cleared
// after the copy instead.
clearMetadata := false
if input.MetadataDirective == types.MetadataDirectiveReplace {
meta := input.Metadata
if meta == nil {
meta = make(map[string]string)
}
if getString(input.Expires) != "" {
meta[string(keyExpires)] = *input.Expires
}
if getString(input.WebsiteRedirectLocation) != "" {
meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation
}
opts.Metadata = parseMetadata(meta)
} else {
// MetadataDirective COPY: StartCopyFromURL would otherwise copy the
// source blob's metadata verbatim, including the internal website-redirect
// key. Set the metadata explicitly so that key is dropped from the
// destination, matching the download+reupload fallback.
if srcProps == nil {
return s3response.CopyObjectOutput{}, fmt.Errorf(
"%w: source properties required to filter metadata", errServerSideCopyFallback)
}
if meta := parseAzMetadata(srcProps.Metadata); meta != nil {
delete(meta, string(keyWebsiteRedirect))
opts.Metadata = parseMetadata(meta)
clearMetadata = len(meta) == 0
}
}
if input.TaggingDirective == types.TaggingDirectiveReplace {
tags, err := backend.ParseObjectTags(getString(input.Tagging))
if err != nil {
return s3response.CopyObjectOutput{}, err
}
opts.BlobTags = tags
}
startResp, err := dstClient.StartCopyFromURL(ctx, srcURL, opts)
if err != nil {
return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err)
}
finalProps, err := az.waitForCopy(ctx, dstClient, startResp.CopyStatus)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
if clearMetadata {
res, err := dstClient.SetMetadata(ctx, nil, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
if res.LastModified != nil {
finalProps.LastModified = res.LastModified
}
if res.ETag != nil {
finalProps.ETag = res.ETag
}
}
if input.MetadataDirective == types.MetadataDirectiveReplace {
res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
BlobCacheControl: input.CacheControl,
BlobContentDisposition: input.ContentDisposition,
BlobContentEncoding: input.ContentEncoding,
BlobContentLanguage: input.ContentLanguage,
BlobContentType: input.ContentType,
}, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
if res.LastModified != nil {
finalProps.LastModified = res.LastModified
}
if res.ETag != nil {
finalProps.ETag = res.ETag
}
}
if input.TaggingDirective == types.TaggingDirectiveCopy {
res, err := srcClient.GetTags(ctx, nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
if err != nil {
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
}
}
if err := az.applyCopyObjectLock(ctx, *input.Bucket, *input.Key, input); err != nil {
return s3response.CopyObjectOutput{}, err
}
var etag string
if finalProps.ETag != nil {
etag = convertAzureEtag(finalProps.ETag)
} else if startResp.ETag != nil {
etag = convertAzureEtag(startResp.ETag)
}
lastModified := finalProps.LastModified
if lastModified == nil {
lastModified = startResp.LastModified
}
return s3response.CopyObjectOutput{
CopyObjectResult: &s3response.CopyObjectResult{
LastModified: lastModified,
ETag: backend.GetPtrFromString(etag),
},
}, nil
}
func (az *Azure) waitForCopy(ctx context.Context, dstClient *blob.Client, initialStatus *blob.CopyStatusType) (*blob.GetPropertiesResponse, error) {
status := initialStatus
for status != nil && *status == blob.CopyStatusTypePending {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(copyPollInterval):
}
props, err := dstClient.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
status = props.CopyStatus
}
props, err := dstClient.GetProperties(ctx, nil)
if err != nil {
return nil, azureErrToS3Err(err)
}
if props.CopyStatus != nil {
switch *props.CopyStatus {
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
return nil, fmt.Errorf("blob copy failed with status %s", *props.CopyStatus)
}
}
return &props, nil
}
func (az *Azure) applyCopyObjectLock(ctx context.Context, bucket, key string, input s3response.CopyObjectInput) error {
if input.ObjectLockLegalHoldStatus != "" {
err := az.PutObjectLegalHold(ctx, bucket, key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return azureErrToS3Err(err)
}
}
if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
retention := s3response.PutObjectRetentionInput{
Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
RetainUntilDate: s3response.AmzDate{
Time: *input.ObjectLockRetainUntilDate,
},
}
retParsed, err := json.Marshal(retention)
if err != nil {
return fmt.Errorf("parse object retention: %w", err)
}
err = az.PutObjectRetention(ctx, bucket, key, "", retParsed)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
}
return azureErrToS3Err(err)
}
}
return nil
}
+162
View File
@@ -0,0 +1,162 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package azure
import (
"context"
"crypto/rand"
"encoding/base64"
"errors"
"net/url"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/versity/versitygw/s3response"
)
const testServiceURL = "https://devstoreaccount1.blob.core.windows.net/devstoreaccount1"
func testSharedKeyAzure(t *testing.T) *Azure {
t.Helper()
// Any valid base64 key will do: the SAS is signed and inspected locally and
// never sent anywhere, so there is no need for a real account key.
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
t.Fatalf("generate account key: %v", err)
}
cred, err := azblob.NewSharedKeyCredential("devstoreaccount1",
base64.StdEncoding.EncodeToString(raw))
if err != nil {
t.Fatalf("NewSharedKeyCredential: %v", err)
}
return &Azure{
serviceURL: testServiceURL,
sharedkeyCreds: cred,
}
}
// copySourceURL returns plain errors: the fallback classification is applied
// once, by its caller. Keeping it in a single place is what puts every failure
// to build a source URL on the fallback path, including the ones that only
// surface in production (GetUserDelegationCredential when the gateway identity
// lacks the Storage Blob Delegator role, or against an endpoint with no
// delegation key API).
func TestCopySourceURLErrorsAreNotPreClassified(t *testing.T) {
az := &Azure{serviceURL: testServiceURL}
_, err := az.copySourceURL(context.Background(), "src-bucket", "src-object")
if err == nil {
t.Fatal("expected an error when no credentials are configured")
}
if errors.Is(err, errServerSideCopyFallback) {
t.Fatal("copySourceURL must not classify its own errors; its caller does")
}
}
// The other half of that contract: an unsignable copy source has to reach the
// caller classified as "server-side copy unavailable", so that CopyObject falls
// back to download+reupload instead of failing the request.
func TestServerSideCopyObjectNoCredentialsFallsBack(t *testing.T) {
az := &Azure{serviceURL: testServiceURL}
_, err := az.serverSideCopyObject(context.Background(),
s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil)
if err == nil {
t.Fatal("expected an error when no credentials are configured")
}
if !errors.Is(err, errServerSideCopyFallback) {
t.Fatalf("error must be classified as a fallback, got %v", err)
}
}
// The MetadataDirective COPY path reads the source properties to filter the
// internal website-redirect key out of the destination metadata, so a caller
// that omits them must be turned away rather than panicking. Falling back keeps
// the copy correct, and skipping the filter instead would silently reintroduce
// the leaked redirect.
func TestServerSideCopyObjectNilSourcePropsFallsBack(t *testing.T) {
az := testSharedKeyAzure(t)
_, err := az.serverSideCopyObject(context.Background(),
s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil)
if err == nil {
t.Fatal("expected an error when the source properties are missing")
}
if !errors.Is(err, errServerSideCopyFallback) {
t.Fatalf("error must be classified as a fallback, got %v", err)
}
}
// The copy-source SAS has to be signed, read-only and scoped to the source
// blob. Its service version is pinned by configuration for endpoints that lag
// the SDK, and left at the SDK default otherwise; an override that silently
// stopped being applied would not fail the integration suite, because copies
// would fall back to download+reupload rather than error.
func TestCopySourceURLSharedKeySAS(t *testing.T) {
for _, tc := range []struct {
name string
version string
}{
{name: "sdk default version", version: ""},
{name: "pinned version", version: "2025-11-05"},
} {
t.Run(tc.name, func(t *testing.T) {
az := testSharedKeyAzure(t)
az.copySASVersion = tc.version
got, err := az.copySourceURL(context.Background(), "src-bucket", "src-object")
if err != nil {
t.Fatalf("copySourceURL: %v", err)
}
base, query, ok := strings.Cut(got, "?")
if !ok {
t.Fatalf("expected a query string carrying the SAS, got %v", got)
}
if want := az.getBlobURL("src-bucket", "src-object"); base != want {
t.Fatalf("expected blob URL %v, got %v", want, base)
}
vals, err := url.ParseQuery(query)
if err != nil {
t.Fatalf("ParseQuery: %v", err)
}
if vals.Get("sig") == "" {
t.Error("expected a signature in the SAS")
}
if got := vals.Get("sp"); got != "r" {
t.Errorf("expected read-only permissions, got %q", got)
}
if got := vals.Get("sr"); got != "b" {
t.Errorf("expected a blob-scoped SAS, got %q", got)
}
sv := vals.Get("sv")
if tc.version == "" {
if sv == "" {
t.Error("expected the SDK default service version to be filled in")
}
return
}
if sv != tc.version {
t.Errorf("expected service version %q, got %q", tc.version, sv)
}
})
}
}
+8 -2
View File
@@ -22,7 +22,7 @@ import (
)
var (
azAccount, azKey, azServiceURL, azSASToken string
azAccount, azKey, azServiceURL, azSASToken, azCopySASVersion string
)
// AzureCommand returns the "azure" subcommand, common to all versitygw
@@ -62,12 +62,18 @@ func AzureCommand() *cli.Command {
Aliases: []string{"u"},
Destination: &azServiceURL,
},
&cli.StringFlag{
Name: "copy-sas-version",
Usage: "service version used to sign the copy-source SAS for server-side copy (defaults to the SDK version; set this for endpoints that lag the SDK, e.g. Azurite)",
EnvVars: []string{"AZ_COPY_SAS_VERSION"},
Destination: &azCopySASVersion,
},
},
}
}
func runAzure(ctx *cli.Context) error {
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold)
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold, azCopySASVersion)
if err != nil {
return fmt.Errorf("init azure: %w", err)
}
+9
View File
@@ -28,6 +28,10 @@ services:
- "10002:10002"
restart: always
hostname: azurite
# Server-side copy makes Azurite fetch the copy source from its own HTTPS
# endpoint; trust the self-signed test cert so that TLS validation succeeds.
environment:
NODE_EXTRA_CA_CERTS: /tests/certs/azurite.pem
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0 --skipApiVersionCheck"
volumes:
- ./tests/certs:/tests/certs
@@ -39,4 +43,9 @@ services:
- ./:/app
ports:
- 7070:7070
# Azurite lags the SDK's SAS version and can't verify a copy-source SAS signed
# with it, so pin the copy-source SAS to a version Azurite supports. Production
# leaves this unset and uses the SDK default.
environment:
AZ_COPY_SAS_VERSION: "2025-11-05"
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
+150
View File
@@ -729,6 +729,57 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error {
})
}
func CopyObject_should_not_copy_website_redirect_without_user_metadata(s *S3Conf) error {
testName := "CopyObject_should_not_copy_website_redirect_without_user_metadata"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
srcObj, dstObj := "source-object", "dest-object"
redirectLocation := "/source-redirect"
// The redirect location is the only thing set on the source: no user
// metadata and no Expires. Backends that keep it alongside user metadata
// have to drop it on copy, and the filtered set is empty here, so a
// backend whose copy treats "empty metadata" the same as "metadata not
// specified" will silently inherit the source's redirect instead.
_, err := putObjectWithData(int64(100), &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
WebsiteRedirectLocation: &redirectLocation,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: &dstObj,
CopySource: getPtr(bucket + "/" + srcObj),
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &dstObj,
})
cancel()
if err != nil {
return err
}
if got := getString(out.WebsiteRedirectLocation); got != "" {
return fmt.Errorf("expected WebsiteRedirectLocation not to be copied, got %v", got)
}
if len(out.Metadata) != 0 {
return fmt.Errorf("expected no user metadata on the destination, instead got %v", out.Metadata)
}
return nil
})
}
func CopyObject_should_replace_meta_props(s *S3Conf) error {
testName := "CopyObject_should_replace_meta_props"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -1622,6 +1673,105 @@ func CopyObject_success(s *S3Conf) error {
})
}
// CopyObject_cross_bucket_server_side_copy exercises a cross-bucket copy where
// source and destination live in different buckets. On the Azure backend this
// drives the server-side StartCopyFromURL path (added in copy.go), verifying that
// object data, user metadata, content-type and tags all survive the copy and that
// an ETag is returned. It also runs on the other backends as a plain copy.
func CopyObject_cross_bucket_server_side_copy(s *S3Conf) error {
testName := "CopyObject_cross_bucket_server_side_copy"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
srcObj, dstObj := "source-object", "dest-object"
dstBucket := getBucketName()
if err := setup(s, dstBucket); err != nil {
return err
}
dataLength := int64(1234567)
cType := "application/json"
meta := map[string]string{
"foo": "bar",
"baz": "quxx",
}
tagging := "foo=bar&baz=quxx"
r, err := putObjectWithData(dataLength, &s3.PutObjectInput{
Bucket: &bucket,
Key: &srcObj,
ContentType: &cType,
Metadata: meta,
Tagging: &tagging,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
copyOut, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &dstBucket,
Key: &dstObj,
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
})
cancel()
if err != nil {
return err
}
if copyOut.CopyObjectResult == nil || getString(copyOut.CopyObjectResult.ETag) == "" {
return fmt.Errorf("expected non-empty ETag in copy result")
}
// Object data must be byte-for-byte identical.
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &dstBucket,
Key: &dstObj,
})
if err != nil {
cancel()
return err
}
bdy, err := io.ReadAll(out.Body)
out.Body.Close()
cancel()
if err != nil {
return err
}
if sha256.Sum256(bdy) != r.csum {
return fmt.Errorf("invalid object data after copy")
}
// Content-type and user metadata must be preserved (MetadataDirective COPY).
if err := checkObjectMetaProps(s3client, dstBucket, dstObj, ObjectMetaProps{
ContentLength: dataLength,
ContentType: cType,
Metadata: meta,
}); err != nil {
return err
}
// Tags must be preserved (TaggingDirective COPY).
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
tagRes, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
Bucket: &dstBucket,
Key: &dstObj,
})
cancel()
if err != nil {
return err
}
expectedTagSet := []types.Tag{
{Key: getPtr("foo"), Value: getPtr("bar")},
{Key: getPtr("baz"), Value: getPtr("quxx")},
}
if !areTagsSame(tagRes.TagSet, expectedTagSet) {
return fmt.Errorf("expected the tag set to be %v, instead got %v",
expectedTagSet, tagRes.TagSet)
}
return teardown(s, dstBucket)
})
}
func CopyObject_with_special_characters(s *S3Conf) error {
testName := "CopyObject_with_special_characters"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+4
View File
@@ -370,6 +370,7 @@ func TestCopyObject(ts *TestState) {
ts.Run(CopyObject_invalid_copy_source)
ts.Run(CopyObject_non_existing_dir_object)
ts.Run(CopyObject_should_copy_meta_props)
ts.Run(CopyObject_should_not_copy_website_redirect_without_user_metadata)
ts.Run(CopyObject_should_replace_meta_props)
ts.Run(CopyObject_invalid_website_redirect_location)
ts.Run(CopyObject_default_content_type_with_replace_metadata)
@@ -392,6 +393,7 @@ func TestCopyObject(ts *TestState) {
}
ts.Run(CopyObject_with_special_characters)
ts.Run(CopyObject_success)
ts.Run(CopyObject_cross_bucket_server_side_copy)
ts.Run(CopyObject_incorrect_source_bucket_expected_owner)
}
@@ -2656,6 +2658,7 @@ func GetIntTests() IntTests {
"CopyObject_invalid_copy_source": CopyObject_invalid_copy_source,
"CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object,
"CopyObject_should_copy_meta_props": CopyObject_should_copy_meta_props,
"CopyObject_should_not_copy_website_redirect_without_user_metadata": CopyObject_should_not_copy_website_redirect_without_user_metadata,
"CopyObject_should_replace_meta_props": CopyObject_should_replace_meta_props,
"CopyObject_invalid_website_redirect_location": CopyObject_invalid_website_redirect_location,
"CopyObject_default_content_type_with_replace_metadata": CopyObject_default_content_type_with_replace_metadata,
@@ -2674,6 +2677,7 @@ func GetIntTests() IntTests {
"CopyObject_to_itself_by_replacing_the_checksum": CopyObject_to_itself_by_replacing_the_checksum,
"CopyObject_with_special_characters": CopyObject_with_special_characters,
"CopyObject_success": CopyObject_success,
"CopyObject_cross_bucket_server_side_copy": CopyObject_cross_bucket_server_side_copy,
"CopyObject_incorrect_source_bucket_expected_owner": CopyObject_incorrect_source_bucket_expected_owner,
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,