feat(azure): server-side copy with download+reupload fallback

* feat(azure): implement server-side copy with fallback Add server-side object copy for the Azure backend using StartCopyFromURL, with a fallback to download+reupload when server-side copy is unavailable. The copy logic lives in backend/azure/copy.go and handles metadata, tagging and object lock configurations. The copy-source SAS service version is configurable via the --copy-sas-version flag (AZ_COPY_SAS_VERSION) and defaults to the SDK version, so production is unchanged. Endpoints that lag the SDK's SAS version (e.g. Azurite) cannot verify a SAS signed with the newer version and can set an older one. On a metadata-COPY, the internal website-redirect key is dropped from the destination to match the download+reupload fallback. Testing: - Add CopyObject_cross_bucket_server_side_copy, which copies an object with data, user metadata, content-type and tags across two buckets and verifies all are preserved and an ETag is returned. - Configure the Azurite functional-test gateway with AZ_COPY_SAS_VERSION and let Azurite trust its self-signed test certificate (NODE_EXTRA_CA_CERTS) so it can fetch the copy source from its own HTTPS endpoint, ensuring CI exercises the real server-side copy path instead of always falling back. Signed-off-by: Nils Leger <nils.leger@getflip.com> * docs: update copyright year Signed-off-by: Nils Leger <nils.leger@getflip.com> * fix: always fallback to download+upload whenever there is an error building the server-side copy URL Signed-off-by: Nils Leger <nils.leger@getflip.com> --------- Signed-off-by: Nils Leger <nils.leger@getflip.com>
2026-09-25 17:34:29 +00:00 · 2026-08-27 16:37:28 -07:00
parent 29bf076abc
commit a780f3473a
7 changed files with 674 additions and 4 deletions
@@ -28,6 +28,10 @@ services:
      - "10002:10002"
    restart: always
    hostname: azurite
+    # Server-side copy makes Azurite fetch the copy source from its own HTTPS
+    # endpoint; trust the self-signed test cert so that TLS validation succeeds.
+    environment:
+      NODE_EXTRA_CA_CERTS: /tests/certs/azurite.pem
    command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0 --skipApiVersionCheck"
    volumes:
      - ./tests/certs:/tests/certs
@@ -39,4 +43,9 @@ services:
      - ./:/app
    ports:
      - 7070:7070
+    # Azurite lags the SDK's SAS version and can't verify a copy-source SAS signed
+    # with it, so pin the copy-source SAS to a version Azurite supports. Production
+    # leaves this unset and uses the SDK default.
+    environment:
+      AZ_COPY_SAS_VERSION: "2025-11-05"
    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
@@ -729,6 +729,57 @@ func CopyObject_should_copy_meta_props(s *S3Conf) error {
 	})
 }

+func CopyObject_should_not_copy_website_redirect_without_user_metadata(s *S3Conf) error {
+	testName := "CopyObject_should_not_copy_website_redirect_without_user_metadata"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		srcObj, dstObj := "source-object", "dest-object"
+		redirectLocation := "/source-redirect"
+
+		// The redirect location is the only thing set on the source: no user
+		// metadata and no Expires. Backends that keep it alongside user metadata
+		// have to drop it on copy, and the filtered set is empty here, so a
+		// backend whose copy treats "empty metadata" the same as "metadata not
+		// specified" will silently inherit the source's redirect instead.
+		_, err := putObjectWithData(int64(100), &s3.PutObjectInput{
+			Bucket:                  &bucket,
+			Key:                     &srcObj,
+			WebsiteRedirectLocation: &redirectLocation,
+		}, s3client)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{
+			Bucket:     &bucket,
+			Key:        &dstObj,
+			CopySource: getPtr(bucket + "/" + srcObj),
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: &bucket,
+			Key:    &dstObj,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		if got := getString(out.WebsiteRedirectLocation); got != "" {
+			return fmt.Errorf("expected WebsiteRedirectLocation not to be copied, got %v", got)
+		}
+		if len(out.Metadata) != 0 {
+			return fmt.Errorf("expected no user metadata on the destination, instead got %v", out.Metadata)
+		}
+
+		return nil
+	})
+}
+
 func CopyObject_should_replace_meta_props(s *S3Conf) error {
 	testName := "CopyObject_should_replace_meta_props"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -1622,6 +1673,105 @@ func CopyObject_success(s *S3Conf) error {
 	})
 }

+// CopyObject_cross_bucket_server_side_copy exercises a cross-bucket copy where
+// source and destination live in different buckets. On the Azure backend this
+// drives the server-side StartCopyFromURL path (added in copy.go), verifying that
+// object data, user metadata, content-type and tags all survive the copy and that
+// an ETag is returned. It also runs on the other backends as a plain copy.
+func CopyObject_cross_bucket_server_side_copy(s *S3Conf) error {
+	testName := "CopyObject_cross_bucket_server_side_copy"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		srcObj, dstObj := "source-object", "dest-object"
+		dstBucket := getBucketName()
+		if err := setup(s, dstBucket); err != nil {
+			return err
+		}
+
+		dataLength := int64(1234567)
+		cType := "application/json"
+		meta := map[string]string{
+			"foo": "bar",
+			"baz": "quxx",
+		}
+		tagging := "foo=bar&baz=quxx"
+
+		r, err := putObjectWithData(dataLength, &s3.PutObjectInput{
+			Bucket:      &bucket,
+			Key:         &srcObj,
+			ContentType: &cType,
+			Metadata:    meta,
+			Tagging:     &tagging,
+		}, s3client)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		copyOut, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
+			Bucket:     &dstBucket,
+			Key:        &dstObj,
+			CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, srcObj)),
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		if copyOut.CopyObjectResult == nil || getString(copyOut.CopyObjectResult.ETag) == "" {
+			return fmt.Errorf("expected non-empty ETag in copy result")
+		}
+
+		// Object data must be byte-for-byte identical.
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: &dstBucket,
+			Key:    &dstObj,
+		})
+		if err != nil {
+			cancel()
+			return err
+		}
+		bdy, err := io.ReadAll(out.Body)
+		out.Body.Close()
+		cancel()
+		if err != nil {
+			return err
+		}
+		if sha256.Sum256(bdy) != r.csum {
+			return fmt.Errorf("invalid object data after copy")
+		}
+
+		// Content-type and user metadata must be preserved (MetadataDirective COPY).
+		if err := checkObjectMetaProps(s3client, dstBucket, dstObj, ObjectMetaProps{
+			ContentLength: dataLength,
+			ContentType:   cType,
+			Metadata:      meta,
+		}); err != nil {
+			return err
+		}
+
+		// Tags must be preserved (TaggingDirective COPY).
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		tagRes, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+			Bucket: &dstBucket,
+			Key:    &dstObj,
+		})
+		cancel()
+		if err != nil {
+			return err
+		}
+		expectedTagSet := []types.Tag{
+			{Key: getPtr("foo"), Value: getPtr("bar")},
+			{Key: getPtr("baz"), Value: getPtr("quxx")},
+		}
+		if !areTagsSame(tagRes.TagSet, expectedTagSet) {
+			return fmt.Errorf("expected the tag set to be %v, instead got %v",
+				expectedTagSet, tagRes.TagSet)
+		}
+
+		return teardown(s, dstBucket)
+	})
+}
+
 func CopyObject_with_special_characters(s *S3Conf) error {
 	testName := "CopyObject_with_special_characters"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -370,6 +370,7 @@ func TestCopyObject(ts *TestState) {
 	ts.Run(CopyObject_invalid_copy_source)
 	ts.Run(CopyObject_non_existing_dir_object)
 	ts.Run(CopyObject_should_copy_meta_props)
+	ts.Run(CopyObject_should_not_copy_website_redirect_without_user_metadata)
 	ts.Run(CopyObject_should_replace_meta_props)
 	ts.Run(CopyObject_invalid_website_redirect_location)
 	ts.Run(CopyObject_default_content_type_with_replace_metadata)
@@ -392,6 +393,7 @@ func TestCopyObject(ts *TestState) {
 	}
 	ts.Run(CopyObject_with_special_characters)
 	ts.Run(CopyObject_success)
+	ts.Run(CopyObject_cross_bucket_server_side_copy)
 	ts.Run(CopyObject_incorrect_source_bucket_expected_owner)
 }

@@ -2656,6 +2658,7 @@ func GetIntTests() IntTests {
 		"CopyObject_invalid_copy_source":                                                   CopyObject_invalid_copy_source,
 		"CopyObject_non_existing_dir_object":                                               CopyObject_non_existing_dir_object,
 		"CopyObject_should_copy_meta_props":                                                CopyObject_should_copy_meta_props,
+		"CopyObject_should_not_copy_website_redirect_without_user_metadata":                CopyObject_should_not_copy_website_redirect_without_user_metadata,
 		"CopyObject_should_replace_meta_props":                                             CopyObject_should_replace_meta_props,
 		"CopyObject_invalid_website_redirect_location":                                     CopyObject_invalid_website_redirect_location,
 		"CopyObject_default_content_type_with_replace_metadata":                            CopyObject_default_content_type_with_replace_metadata,
@@ -2674,6 +2677,7 @@ func GetIntTests() IntTests {
 		"CopyObject_to_itself_by_replacing_the_checksum":                                   CopyObject_to_itself_by_replacing_the_checksum,
 		"CopyObject_with_special_characters":                                               CopyObject_with_special_characters,
 		"CopyObject_success":                                                               CopyObject_success,
+		"CopyObject_cross_bucket_server_side_copy":                                         CopyObject_cross_bucket_server_side_copy,
 		"CopyObject_incorrect_source_bucket_expected_owner":                                CopyObject_incorrect_source_bucket_expected_owner,
 		"PutObjectTagging_non_existing_object":                                             PutObjectTagging_non_existing_object,
 		"PutObjectTagging_long_tags":                                                       PutObjectTagging_long_tags,