fix: enforce 5gb copy source object size threshold.

Fixes #1896

Enforces the S3 `5 GiB` copy source size limit across the posix and azure
backends for `CopyObject` and `UploadPartCopy`, returning `InvalidRequest` when
the source object exceeds the threshold.

The limit is now configurable via `--copy-object-threshold`
(`VGW_COPY_OBJECT_THRESHOLD`, default 5 GiB).
A new `--mp-max-parts flag` (`VGW_MP_MAX_PARTS`, default `10000`) has been added to make multipart upload parts number limit configurable.

No integration test has been added, as GitHub Actions cannot reliably
handle large objects.

cmd/versitygw/azure.go

@@ -65,7 +65,7 @@ func azureCommand() *cli.Command {
 }
 
 func runAzure(ctx *cli.Context) error {
-	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
+	be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken, copyObjectThreshold)
 	if err != nil {
 		return fmt.Errorf("init azure: %w", err)
 	}

cmd/versitygw/gateway_test.go

@@ -34,6 +34,8 @@ func initEnv(dir string) {
 	maxConnections = 250000
 	maxRequests = 100000
 	ports = []string{"127.0.0.1:7070"}
+	mpMaxParts = 10000
+	copyObjectThreshold = 5 * 1024 * 1024 * 1024
 
 	// client
 	awsID = "user"

cmd/versitygw/main.go

@@ -103,6 +103,8 @@ var (
 	webuiPathPrefix                        string
 	webuiS3Prefix                          string
 	disableACLs                            bool
+	mpMaxParts                             int
+	copyObjectThreshold                    int64
 )
 
 var (
@@ -755,6 +757,20 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IPA_INSECURE"},
 			Destination: &ipaInsecure,
 		},
+		&cli.IntFlag{
+			Name:        "mp-max-parts",
+			Usage:       "maximum number of parts allowed in a multipart upload",
+			EnvVars:     []string{"VGW_MP_MAX_PARTS"},
+			Value:       10000,
+			Destination: &mpMaxParts,
+		},
+		&cli.Int64Flag{
+			Name:        "copy-object-threshold",
+			Usage:       "maximum allowed source object size in bytes for CopyObject; objects larger than this are rejected",
+			EnvVars:     []string{"VGW_COPY_OBJECT_THRESHOLD"},
+			Value:       5 * 1024 * 1024 * 1024,
+			Destination: &copyObjectThreshold,
+		},
 	}
 }
 
@@ -778,6 +794,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		log.Printf("WARNING: max-requests (%d) exceeds max-connections (%d) which could allow for gateway to panic before throttling requests",
 			maxRequests, maxConnections)
 	}
+	if mpMaxParts < 1 {
+		return fmt.Errorf("mp-max-parts must be positive")
+	}
+	if copyObjectThreshold < 1 {
+		return fmt.Errorf("copy-object-threshold must be positive")
+	}
 
 	// Ensure we have at least one port specified
 	if len(ports) == 0 {
@@ -842,6 +864,7 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 
 	opts := []s3api.Option{
 		s3api.WithConcurrencyLimiter(maxConnections, maxRequests),
+		s3api.WithMpMaxParts(mpMaxParts),
 	}
 	if corsAllowOrigin != "" {
 		opts = append(opts, s3api.WithCORSAllowOrigin(corsAllowOrigin))

cmd/versitygw/posix.go

@@ -148,6 +148,7 @@ func runPosix(ctx *cli.Context) error {
 		ForceNoCopyFileRange: forceNoCopyFileRange,
 		ValidateBucketNames:  disableStrictBucketNames,
 		Concurrency:          actionsConcurrency,
+		CopyObjectThreshold:  copyObjectThreshold,
 	}
 
 	var ms meta.MetadataStorer

cmd/versitygw/scoutfs.go

@@ -134,6 +134,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ValidateBucketNames = disableStrictBucketNames
 	opts.SetProjectID = setProjectID
 	opts.Concurrency = actionsConcurrency
+	opts.CopyObjectThreshold = copyObjectThreshold
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {