mirror of
https://github.com/versity/versitygw.git
synced 2026-08-28 20:06:02 +00:00
feat(azure): server-side copy with download+reupload fallback
* feat(azure): implement server-side copy with fallback Add server-side object copy for the Azure backend using StartCopyFromURL, with a fallback to download+reupload when server-side copy is unavailable. The copy logic lives in backend/azure/copy.go and handles metadata, tagging and object lock configurations. The copy-source SAS service version is configurable via the --copy-sas-version flag (AZ_COPY_SAS_VERSION) and defaults to the SDK version, so production is unchanged. Endpoints that lag the SDK's SAS version (e.g. Azurite) cannot verify a SAS signed with the newer version and can set an older one. On a metadata-COPY, the internal website-redirect key is dropped from the destination to match the download+reupload fallback. Testing: - Add CopyObject_cross_bucket_server_side_copy, which copies an object with data, user metadata, content-type and tags across two buckets and verifies all are preserved and an ETag is returned. - Configure the Azurite functional-test gateway with AZ_COPY_SAS_VERSION and let Azurite trust its self-signed test certificate (NODE_EXTRA_CA_CERTS) so it can fetch the copy source from its own HTTPS endpoint, ensuring CI exercises the real server-side copy path instead of always falling back. Signed-off-by: Nils Leger <nils.leger@getflip.com> * docs: update copyright year Signed-off-by: Nils Leger <nils.leger@getflip.com> * fix: always fallback to download+upload whenever there is an error building the server-side copy URL Signed-off-by: Nils Leger <nils.leger@getflip.com> --------- Signed-off-by: Nils Leger <nils.leger@getflip.com>
This commit is contained in:
+37
-2
@@ -47,6 +47,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/debuglogger"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
@@ -110,11 +111,16 @@ type Azure struct {
|
||||
serviceURL string
|
||||
sasToken string
|
||||
copyObjectThreshold int64
|
||||
// copySASVersion overrides the service version used to sign the copy-source
|
||||
// SAS for server-side copy. Empty means use the SDK default. This exists so
|
||||
// endpoints that lag the SDK's SAS version (e.g. Azurite) can verify the
|
||||
// signature; production leaves it unset.
|
||||
copySASVersion string
|
||||
}
|
||||
|
||||
var _ backend.Backend = &Azure{}
|
||||
|
||||
func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64) (*Azure, error) {
|
||||
func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThreshold int64, copySASVersion string) (*Azure, error) {
|
||||
url := serviceURL
|
||||
if serviceURL == "" && accountName != "" {
|
||||
// if not otherwise specified, use the typical form:
|
||||
@@ -132,6 +138,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
|
||||
serviceURL: serviceURL,
|
||||
sasToken: sasToken,
|
||||
copyObjectThreshold: copyObjectThreshold,
|
||||
copySASVersion: copySASVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -154,6 +161,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
|
||||
serviceURL: url,
|
||||
defaultCreds: cred,
|
||||
copyObjectThreshold: copyObjectThreshold,
|
||||
copySASVersion: copySASVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -172,6 +180,7 @@ func New(accountName, accountKey, serviceURL, sasToken string, copyObjectThresho
|
||||
serviceURL: url,
|
||||
sharedkeyCreds: cred,
|
||||
copyObjectThreshold: copyObjectThreshold,
|
||||
copySASVersion: copySASVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1252,7 +1261,33 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get the source object
|
||||
// Cross-object copy: try server-side copy first, fall back to download+reupload.
|
||||
srcClient, err := az.getBlobClient(srcBucket, srcObj)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
srcProps, err := srcClient.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
}
|
||||
|
||||
if srcProps.ContentLength != nil && *srcProps.ContentLength > az.copyObjectThreshold {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetCopySourceObjectTooLargeErr(az.copyObjectThreshold)
|
||||
}
|
||||
|
||||
out, err := az.serverSideCopyObject(ctx, input, srcBucket, srcObj, dstClient, srcClient, &srcProps)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
if !errors.Is(err, errServerSideCopyFallback) {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
debuglogger.Logf("falling back to download+reupload (%q/%q -> %q/%q): %v",
|
||||
srcBucket, srcObj, *input.Bucket, *input.Key, err)
|
||||
|
||||
// Fallback: download and re-upload through the gateway.
|
||||
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
const copyPollInterval = 500 * time.Millisecond
|
||||
|
||||
// errServerSideCopyFallback signals that server-side copy could not be started
|
||||
// and the caller should fall back to download+reupload. Failures before the
|
||||
// copy is started are wrapped in this sentinel; once StartCopyFromURL has
|
||||
// succeeded the destination blob has been written, so later failures are
|
||||
// returned as-is rather than silently re-copying through the gateway.
|
||||
var errServerSideCopyFallback = errors.New("server-side copy unavailable")
|
||||
|
||||
func (az *Azure) copySourceURL(ctx context.Context, srcBucket, srcObj string) (string, error) {
|
||||
if az.sharedkeyCreds != nil {
|
||||
now := time.Now().UTC()
|
||||
sasQueryParams, err := sas.BlobSignatureValues{
|
||||
Protocol: sas.ProtocolHTTPS,
|
||||
Version: az.copySASVersion,
|
||||
StartTime: now.Add(-10 * time.Second),
|
||||
ExpiryTime: now.Add(15 * time.Minute),
|
||||
Permissions: (&sas.BlobPermissions{Read: true}).String(),
|
||||
ContainerName: srcBucket,
|
||||
BlobName: srcObj,
|
||||
}.SignWithSharedKey(az.sharedkeyCreds)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil
|
||||
}
|
||||
|
||||
if az.sasToken != "" {
|
||||
return az.getBlobURL(srcBucket, srcObj) + "?" + az.sasToken, nil
|
||||
}
|
||||
|
||||
if az.defaultCreds != nil {
|
||||
svcClient, err := service.NewClient(az.serviceURL, az.defaultCreds, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("init service client: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
info := service.KeyInfo{
|
||||
Start: backend.GetPtrFromString(now.Add(-10 * time.Second).Format(sas.TimeFormat)),
|
||||
Expiry: backend.GetPtrFromString(now.Add(48 * time.Hour).Format(sas.TimeFormat)),
|
||||
}
|
||||
|
||||
udc, err := svcClient.GetUserDelegationCredential(ctx, info, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get user delegation credential: %w", err)
|
||||
}
|
||||
|
||||
perms := &sas.BlobPermissions{Read: true}
|
||||
sasQueryParams, err := sas.BlobSignatureValues{
|
||||
Protocol: sas.ProtocolHTTPS,
|
||||
Version: az.copySASVersion,
|
||||
StartTime: now.Add(-10 * time.Second),
|
||||
ExpiryTime: now.Add(15 * time.Minute),
|
||||
Permissions: perms.String(),
|
||||
ContainerName: srcBucket,
|
||||
BlobName: srcObj,
|
||||
}.SignWithUserDelegation(udc)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("sign user delegation sas: %w", err)
|
||||
}
|
||||
|
||||
return az.getBlobURL(srcBucket, srcObj) + "?" + sasQueryParams.Encode(), nil
|
||||
}
|
||||
|
||||
return "", errors.New("no credentials available")
|
||||
}
|
||||
|
||||
func (az *Azure) serverSideCopyObject(
|
||||
ctx context.Context,
|
||||
input s3response.CopyObjectInput,
|
||||
srcBucket, srcObj string,
|
||||
dstClient, srcClient *blob.Client,
|
||||
srcProps *blob.GetPropertiesResponse,
|
||||
) (s3response.CopyObjectOutput, error) {
|
||||
srcURL, err := az.copySourceURL(ctx, srcBucket, srcObj)
|
||||
if err != nil {
|
||||
// Any failure to build a signed source URL means server-side copy can't
|
||||
// be started at all, so fall back instead of failing the request. This
|
||||
// notably covers GetUserDelegationCredential, which fails if the gateway
|
||||
// identity lacks the Storage Blob Delegator role or the endpoint doesn't
|
||||
// implement the delegation key API.
|
||||
return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err)
|
||||
}
|
||||
|
||||
opts := &blob.StartCopyFromURLOptions{}
|
||||
|
||||
// Copy Blob reads absent x-ms-meta-* headers as "inherit the source
|
||||
// metadata", so an empty opts.Metadata cannot express "no metadata". When
|
||||
// filtering empties the set, the destination metadata has to be cleared
|
||||
// after the copy instead.
|
||||
clearMetadata := false
|
||||
|
||||
if input.MetadataDirective == types.MetadataDirectiveReplace {
|
||||
meta := input.Metadata
|
||||
if meta == nil {
|
||||
meta = make(map[string]string)
|
||||
}
|
||||
if getString(input.Expires) != "" {
|
||||
meta[string(keyExpires)] = *input.Expires
|
||||
}
|
||||
if getString(input.WebsiteRedirectLocation) != "" {
|
||||
meta[string(keyWebsiteRedirect)] = *input.WebsiteRedirectLocation
|
||||
}
|
||||
opts.Metadata = parseMetadata(meta)
|
||||
} else {
|
||||
// MetadataDirective COPY: StartCopyFromURL would otherwise copy the
|
||||
// source blob's metadata verbatim, including the internal website-redirect
|
||||
// key. Set the metadata explicitly so that key is dropped from the
|
||||
// destination, matching the download+reupload fallback.
|
||||
if srcProps == nil {
|
||||
return s3response.CopyObjectOutput{}, fmt.Errorf(
|
||||
"%w: source properties required to filter metadata", errServerSideCopyFallback)
|
||||
}
|
||||
if meta := parseAzMetadata(srcProps.Metadata); meta != nil {
|
||||
delete(meta, string(keyWebsiteRedirect))
|
||||
opts.Metadata = parseMetadata(meta)
|
||||
clearMetadata = len(meta) == 0
|
||||
}
|
||||
}
|
||||
|
||||
if input.TaggingDirective == types.TaggingDirectiveReplace {
|
||||
tags, err := backend.ParseObjectTags(getString(input.Tagging))
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
opts.BlobTags = tags
|
||||
}
|
||||
|
||||
startResp, err := dstClient.StartCopyFromURL(ctx, srcURL, opts)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, fmt.Errorf("%w: %v", errServerSideCopyFallback, err)
|
||||
}
|
||||
|
||||
finalProps, err := az.waitForCopy(ctx, dstClient, startResp.CopyStatus)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
if clearMetadata {
|
||||
res, err := dstClient.SetMetadata(ctx, nil, nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
}
|
||||
if res.LastModified != nil {
|
||||
finalProps.LastModified = res.LastModified
|
||||
}
|
||||
if res.ETag != nil {
|
||||
finalProps.ETag = res.ETag
|
||||
}
|
||||
}
|
||||
|
||||
if input.MetadataDirective == types.MetadataDirectiveReplace {
|
||||
res, err := dstClient.SetHTTPHeaders(ctx, blob.HTTPHeaders{
|
||||
BlobCacheControl: input.CacheControl,
|
||||
BlobContentDisposition: input.ContentDisposition,
|
||||
BlobContentEncoding: input.ContentEncoding,
|
||||
BlobContentLanguage: input.ContentLanguage,
|
||||
BlobContentType: input.ContentType,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
}
|
||||
if res.LastModified != nil {
|
||||
finalProps.LastModified = res.LastModified
|
||||
}
|
||||
if res.ETag != nil {
|
||||
finalProps.ETag = res.ETag
|
||||
}
|
||||
}
|
||||
|
||||
if input.TaggingDirective == types.TaggingDirectiveCopy {
|
||||
res, err := srcClient.GetTags(ctx, nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
}
|
||||
_, err = dstClient.SetTags(ctx, parseAzTags(res.BlobTagSet), nil)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, azureErrToS3Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := az.applyCopyObjectLock(ctx, *input.Bucket, *input.Key, input); err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
var etag string
|
||||
if finalProps.ETag != nil {
|
||||
etag = convertAzureEtag(finalProps.ETag)
|
||||
} else if startResp.ETag != nil {
|
||||
etag = convertAzureEtag(startResp.ETag)
|
||||
}
|
||||
|
||||
lastModified := finalProps.LastModified
|
||||
if lastModified == nil {
|
||||
lastModified = startResp.LastModified
|
||||
}
|
||||
|
||||
return s3response.CopyObjectOutput{
|
||||
CopyObjectResult: &s3response.CopyObjectResult{
|
||||
LastModified: lastModified,
|
||||
ETag: backend.GetPtrFromString(etag),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (az *Azure) waitForCopy(ctx context.Context, dstClient *blob.Client, initialStatus *blob.CopyStatusType) (*blob.GetPropertiesResponse, error) {
|
||||
status := initialStatus
|
||||
for status != nil && *status == blob.CopyStatusTypePending {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(copyPollInterval):
|
||||
}
|
||||
|
||||
props, err := dstClient.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, azureErrToS3Err(err)
|
||||
}
|
||||
status = props.CopyStatus
|
||||
}
|
||||
|
||||
props, err := dstClient.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, azureErrToS3Err(err)
|
||||
}
|
||||
|
||||
if props.CopyStatus != nil {
|
||||
switch *props.CopyStatus {
|
||||
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
|
||||
return nil, fmt.Errorf("blob copy failed with status %s", *props.CopyStatus)
|
||||
}
|
||||
}
|
||||
|
||||
return &props, nil
|
||||
}
|
||||
|
||||
func (az *Azure) applyCopyObjectLock(ctx context.Context, bucket, key string, input s3response.CopyObjectInput) error {
|
||||
if input.ObjectLockLegalHoldStatus != "" {
|
||||
err := az.PutObjectLegalHold(ctx, bucket, key, "", input.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
|
||||
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
|
||||
}
|
||||
return azureErrToS3Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
if input.ObjectLockMode != "" && input.ObjectLockRetainUntilDate != nil {
|
||||
retention := s3response.PutObjectRetentionInput{
|
||||
Mode: types.ObjectLockRetentionMode(input.ObjectLockMode),
|
||||
RetainUntilDate: s3response.AmzDate{
|
||||
Time: *input.ObjectLockRetainUntilDate,
|
||||
},
|
||||
}
|
||||
|
||||
retParsed, err := json.Marshal(retention)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse object retention: %w", err)
|
||||
}
|
||||
err = az.PutObjectRetention(ctx, bucket, key, "", retParsed)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrMissingObjectLockConfiguration)) {
|
||||
err = s3err.GetAPIError(s3err.ErrMissingObjectLockConfigurationNoSpaces)
|
||||
}
|
||||
return azureErrToS3Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
const testServiceURL = "https://devstoreaccount1.blob.core.windows.net/devstoreaccount1"
|
||||
|
||||
func testSharedKeyAzure(t *testing.T) *Azure {
|
||||
t.Helper()
|
||||
|
||||
// Any valid base64 key will do: the SAS is signed and inspected locally and
|
||||
// never sent anywhere, so there is no need for a real account key.
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
t.Fatalf("generate account key: %v", err)
|
||||
}
|
||||
|
||||
cred, err := azblob.NewSharedKeyCredential("devstoreaccount1",
|
||||
base64.StdEncoding.EncodeToString(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("NewSharedKeyCredential: %v", err)
|
||||
}
|
||||
|
||||
return &Azure{
|
||||
serviceURL: testServiceURL,
|
||||
sharedkeyCreds: cred,
|
||||
}
|
||||
}
|
||||
|
||||
// copySourceURL returns plain errors: the fallback classification is applied
|
||||
// once, by its caller. Keeping it in a single place is what puts every failure
|
||||
// to build a source URL on the fallback path, including the ones that only
|
||||
// surface in production (GetUserDelegationCredential when the gateway identity
|
||||
// lacks the Storage Blob Delegator role, or against an endpoint with no
|
||||
// delegation key API).
|
||||
func TestCopySourceURLErrorsAreNotPreClassified(t *testing.T) {
|
||||
az := &Azure{serviceURL: testServiceURL}
|
||||
|
||||
_, err := az.copySourceURL(context.Background(), "src-bucket", "src-object")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when no credentials are configured")
|
||||
}
|
||||
if errors.Is(err, errServerSideCopyFallback) {
|
||||
t.Fatal("copySourceURL must not classify its own errors; its caller does")
|
||||
}
|
||||
}
|
||||
|
||||
// The other half of that contract: an unsignable copy source has to reach the
|
||||
// caller classified as "server-side copy unavailable", so that CopyObject falls
|
||||
// back to download+reupload instead of failing the request.
|
||||
func TestServerSideCopyObjectNoCredentialsFallsBack(t *testing.T) {
|
||||
az := &Azure{serviceURL: testServiceURL}
|
||||
|
||||
_, err := az.serverSideCopyObject(context.Background(),
|
||||
s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when no credentials are configured")
|
||||
}
|
||||
if !errors.Is(err, errServerSideCopyFallback) {
|
||||
t.Fatalf("error must be classified as a fallback, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The MetadataDirective COPY path reads the source properties to filter the
|
||||
// internal website-redirect key out of the destination metadata, so a caller
|
||||
// that omits them must be turned away rather than panicking. Falling back keeps
|
||||
// the copy correct, and skipping the filter instead would silently reintroduce
|
||||
// the leaked redirect.
|
||||
func TestServerSideCopyObjectNilSourcePropsFallsBack(t *testing.T) {
|
||||
az := testSharedKeyAzure(t)
|
||||
|
||||
_, err := az.serverSideCopyObject(context.Background(),
|
||||
s3response.CopyObjectInput{}, "src-bucket", "src-object", nil, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when the source properties are missing")
|
||||
}
|
||||
if !errors.Is(err, errServerSideCopyFallback) {
|
||||
t.Fatalf("error must be classified as a fallback, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The copy-source SAS has to be signed, read-only and scoped to the source
|
||||
// blob. Its service version is pinned by configuration for endpoints that lag
|
||||
// the SDK, and left at the SDK default otherwise; an override that silently
|
||||
// stopped being applied would not fail the integration suite, because copies
|
||||
// would fall back to download+reupload rather than error.
|
||||
func TestCopySourceURLSharedKeySAS(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
version string
|
||||
}{
|
||||
{name: "sdk default version", version: ""},
|
||||
{name: "pinned version", version: "2025-11-05"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
az := testSharedKeyAzure(t)
|
||||
az.copySASVersion = tc.version
|
||||
|
||||
got, err := az.copySourceURL(context.Background(), "src-bucket", "src-object")
|
||||
if err != nil {
|
||||
t.Fatalf("copySourceURL: %v", err)
|
||||
}
|
||||
|
||||
base, query, ok := strings.Cut(got, "?")
|
||||
if !ok {
|
||||
t.Fatalf("expected a query string carrying the SAS, got %v", got)
|
||||
}
|
||||
if want := az.getBlobURL("src-bucket", "src-object"); base != want {
|
||||
t.Fatalf("expected blob URL %v, got %v", want, base)
|
||||
}
|
||||
|
||||
vals, err := url.ParseQuery(query)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseQuery: %v", err)
|
||||
}
|
||||
if vals.Get("sig") == "" {
|
||||
t.Error("expected a signature in the SAS")
|
||||
}
|
||||
if got := vals.Get("sp"); got != "r" {
|
||||
t.Errorf("expected read-only permissions, got %q", got)
|
||||
}
|
||||
if got := vals.Get("sr"); got != "b" {
|
||||
t.Errorf("expected a blob-scoped SAS, got %q", got)
|
||||
}
|
||||
|
||||
sv := vals.Get("sv")
|
||||
if tc.version == "" {
|
||||
if sv == "" {
|
||||
t.Error("expected the SDK default service version to be filled in")
|
||||
}
|
||||
return
|
||||
}
|
||||
if sv != tc.version {
|
||||
t.Errorf("expected service version %q, got %q", tc.version, sv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
azAccount, azKey, azServiceURL, azSASToken string
|
||||
azAccount, azKey, azServiceURL, azSASToken, azCopySASVersion string
|
||||
)
|
||||
|
||||
// AzureCommand returns the "azure" subcommand, common to all versitygw
|
||||
@@ -62,12 +62,18 @@ func AzureCommand() *cli.Command {
|
||||
Aliases: []string{"u"},
|
||||
Destination: &azServiceURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "copy-sas-version",
|
||||
Usage: "service version used to sign the copy-source SAS for server-side copy (defaults to the SDK version; set this for endpoints that lag the SDK, e.g. Azurite)",
|
||||
EnvVars: []string{"AZ_COPY_SAS_VERSION"},
|
||||
Destination: &azCopySASVersion,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runAzure(ctx *cli.Context) error {
|
||||
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold)
|
||||
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, CopyObjectThreshold, azCopySASVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init azure: %w", err)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,10 @@ services:
|
||||
- "10002:10002"
|
||||
restart: always
|
||||
hostname: azurite
|
||||
# Server-side copy makes Azurite fetch the copy source from its own HTTPS
|
||||
# endpoint; trust the self-signed test cert so that TLS validation succeeds.
|
||||
environment:
|
||||
NODE_EXTRA_CA_CERTS: /tests/certs/azurite.pem
|
||||
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0 --skipApiVersionCheck"
|
||||
volumes:
|
||||
- ./tests/certs:/tests/certs
|
||||
@@ -39,4 +43,9 @@ services:
|
||||
- ./:/app
|
||||
ports:
|
||||
- 7070:7070
|
||||
# Azurite lags the SDK's SAS version and can't verify a copy-source SAS signed
|
||||
# with it, so pin the copy-source SAS to a version Azurite supports. Production
|
||||
# leaves this unset and uses the SDK default.
|
||||
environment:
|
||||
AZ_COPY_SAS_VERSION: "2025-11-05"
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
|
||||
@@ -729,6 +729,57 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func CopyObject_should_not_copy_website_redirect_without_user_metadata(s *S3Conf) error {
|
||||
testName := "CopyObject_should_not_copy_website_redirect_without_user_metadata"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
srcObj, dstObj := "source-object", "dest-object"
|
||||
redirectLocation := "/source-redirect"
|
||||
|
||||
// The redirect location is the only thing set on the source: no user
|
||||
// metadata and no Expires. Backends that keep it alongside user metadata
|
||||
// have to drop it on copy, and the filtered set is empty here, so a
|
||||
// backend whose copy treats "empty metadata" the same as "metadata not
|
||||
// specified" will silently inherit the source's redirect instead.
|
||||
_, err := putObjectWithData(int64(100), &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &srcObj,
|
||||
WebsiteRedirectLocation: &redirectLocation,
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &dstObj,
|
||||
CopySource: getPtr(bucket + "/" + srcObj),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &dstObj,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got := getString(out.WebsiteRedirectLocation); got != "" {
|
||||
return fmt.Errorf("expected WebsiteRedirectLocation not to be copied, got %v", got)
|
||||
}
|
||||
if len(out.Metadata) != 0 {
|
||||
return fmt.Errorf("expected no user metadata on the destination, instead got %v", out.Metadata)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CopyObject_should_replace_meta_props(s *S3Conf) error {
|
||||
testName := "CopyObject_should_replace_meta_props"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -1622,6 +1673,105 @@ func CopyObject_success(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
// CopyObject_cross_bucket_server_side_copy exercises a cross-bucket copy where
|
||||
// source and destination live in different buckets. On the Azure backend this
|
||||
// drives the server-side StartCopyFromURL path (added in copy.go), verifying that
|
||||
// object data, user metadata, content-type and tags all survive the copy and that
|
||||
// an ETag is returned. It also runs on the other backends as a plain copy.
|
||||
func CopyObject_cross_bucket_server_side_copy(s *S3Conf) error {
|
||||
testName := "CopyObject_cross_bucket_server_side_copy"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
srcObj, dstObj := "source-object", "dest-object"
|
||||
dstBucket := getBucketName()
|
||||
if err := setup(s, dstBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataLength := int64(1234567)
|
||||
cType := "application/json"
|
||||
meta := map[string]string{
|
||||
"foo": "bar",
|
||||
"baz": "quxx",
|
||||
}
|
||||
tagging := "foo=bar&baz=quxx"
|
||||
|
||||
r, err := putObjectWithData(dataLength, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &srcObj,
|
||||
ContentType: &cType,
|
||||
Metadata: meta,
|
||||
Tagging: &tagging,
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
copyOut, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObj,
|
||||
CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if copyOut.CopyObjectResult == nil || getString(copyOut.CopyObjectResult.ETag) == "" {
|
||||
return fmt.Errorf("expected non-empty ETag in copy result")
|
||||
}
|
||||
|
||||
// Object data must be byte-for-byte identical.
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObj,
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
bdy, err := io.ReadAll(out.Body)
|
||||
out.Body.Close()
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sha256.Sum256(bdy) != r.csum {
|
||||
return fmt.Errorf("invalid object data after copy")
|
||||
}
|
||||
|
||||
// Content-type and user metadata must be preserved (MetadataDirective COPY).
|
||||
if err := checkObjectMetaProps(s3client, dstBucket, dstObj, ObjectMetaProps{
|
||||
ContentLength: dataLength,
|
||||
ContentType: cType,
|
||||
Metadata: meta,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Tags must be preserved (TaggingDirective COPY).
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
tagRes, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObj,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expectedTagSet := []types.Tag{
|
||||
{Key: getPtr("foo"), Value: getPtr("bar")},
|
||||
{Key: getPtr("baz"), Value: getPtr("quxx")},
|
||||
}
|
||||
if !areTagsSame(tagRes.TagSet, expectedTagSet) {
|
||||
return fmt.Errorf("expected the tag set to be %v, instead got %v",
|
||||
expectedTagSet, tagRes.TagSet)
|
||||
}
|
||||
|
||||
return teardown(s, dstBucket)
|
||||
})
|
||||
}
|
||||
|
||||
func CopyObject_with_special_characters(s *S3Conf) error {
|
||||
testName := "CopyObject_with_special_characters"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
|
||||
@@ -370,6 +370,7 @@ func TestCopyObject(ts *TestState) {
|
||||
ts.Run(CopyObject_invalid_copy_source)
|
||||
ts.Run(CopyObject_non_existing_dir_object)
|
||||
ts.Run(CopyObject_should_copy_meta_props)
|
||||
ts.Run(CopyObject_should_not_copy_website_redirect_without_user_metadata)
|
||||
ts.Run(CopyObject_should_replace_meta_props)
|
||||
ts.Run(CopyObject_invalid_website_redirect_location)
|
||||
ts.Run(CopyObject_default_content_type_with_replace_metadata)
|
||||
@@ -392,6 +393,7 @@ func TestCopyObject(ts *TestState) {
|
||||
}
|
||||
ts.Run(CopyObject_with_special_characters)
|
||||
ts.Run(CopyObject_success)
|
||||
ts.Run(CopyObject_cross_bucket_server_side_copy)
|
||||
ts.Run(CopyObject_incorrect_source_bucket_expected_owner)
|
||||
}
|
||||
|
||||
@@ -2656,6 +2658,7 @@ func GetIntTests() IntTests {
|
||||
"CopyObject_invalid_copy_source": CopyObject_invalid_copy_source,
|
||||
"CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object,
|
||||
"CopyObject_should_copy_meta_props": CopyObject_should_copy_meta_props,
|
||||
"CopyObject_should_not_copy_website_redirect_without_user_metadata": CopyObject_should_not_copy_website_redirect_without_user_metadata,
|
||||
"CopyObject_should_replace_meta_props": CopyObject_should_replace_meta_props,
|
||||
"CopyObject_invalid_website_redirect_location": CopyObject_invalid_website_redirect_location,
|
||||
"CopyObject_default_content_type_with_replace_metadata": CopyObject_default_content_type_with_replace_metadata,
|
||||
@@ -2674,6 +2677,7 @@ func GetIntTests() IntTests {
|
||||
"CopyObject_to_itself_by_replacing_the_checksum": CopyObject_to_itself_by_replacing_the_checksum,
|
||||
"CopyObject_with_special_characters": CopyObject_with_special_characters,
|
||||
"CopyObject_success": CopyObject_success,
|
||||
"CopyObject_cross_bucket_server_side_copy": CopyObject_cross_bucket_server_side_copy,
|
||||
"CopyObject_incorrect_source_bucket_expected_owner": CopyObject_incorrect_source_bucket_expected_owner,
|
||||
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
|
||||
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
|
||||
|
||||
Reference in New Issue
Block a user