fix: add gcs compatibility flag to fix s3proxy GCS SigV4 signature mismatch

The AWS SDK v2 includes Accept-Encoding in SigV4 signed headers
which causes GCS to return a SignatureDoesNotMatch error because
GCS rewrites that header internally before verifying the signature.

Add a --gcs-compatibility / VGW_S3_GCS_COMPATIBILITY option for the s3proxy
backend that injects two Smithy finalize-layer middlewares: one removes
Accept-Encoding from the request immediately before the Signing step, and
a second restores it after signing so the header is still sent on the wire.

see: https://github.com/aws/aws-sdk-go-v2/issues/1816

This can be removed once GCS fixes this incompatibility.

backend/s3proxy/client.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
 )
 
 func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
@@ -82,6 +83,17 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 			config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired))
 	}
 
+	if s.gcsCompatibility {
+		opts = append(opts, config.WithAPIOptions([]func(*middleware.Stack) error{
+			func(stack *middleware.Stack) error {
+				if err := stack.Finalize.Insert(gcsIgnoreHeadersMiddleware(), "Signing", middleware.Before); err != nil {
+					return err
+				}
+				return stack.Finalize.Insert(gcsRestoreHeadersMiddleware(), "Signing", middleware.After)
+			},
+		}))
+	}
+
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -89,3 +101,57 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 
 	return config.LoadDefaultConfig(ctx, opts...)
 }
+
+// gcsIgnoredHeadersKey is the context key for headers temporarily removed
+// before signing to work around GCS SigV4 compatibility issue.
+// See: https://github.com/aws/aws-sdk-go-v2/issues/1816
+type gcsIgnoredHeadersKey struct{}
+
+// gcsIgnoreHeadersMiddleware removes Accept-Encoding from the request before
+// the Signing step so it is not included in signed headers. GCS rejects
+// requests where Accept-Encoding is part of the signature because it rewrites
+// that header internally.
+func gcsIgnoreHeadersMiddleware() middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc("GCSIgnoreHeaders",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
+			out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+		) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{
+					Err: fmt.Errorf("(GCSIgnoreHeaders) unexpected request type %T", in.Request),
+				}
+			}
+
+			const hdr = "Accept-Encoding"
+			saved := req.Header.Get(hdr)
+			req.Header.Del(hdr)
+			ctx = middleware.WithStackValue(ctx, gcsIgnoredHeadersKey{}, saved)
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}
+
+// gcsRestoreHeadersMiddleware restores the Accept-Encoding header that was
+// removed by gcsIgnoreHeadersMiddleware so it is still sent on the wire.
+func gcsRestoreHeadersMiddleware() middleware.FinalizeMiddleware {
+	return middleware.FinalizeMiddlewareFunc("GCSRestoreHeaders",
+		func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
+			out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+		) {
+			req, ok := in.Request.(*smithyhttp.Request)
+			if !ok {
+				return out, metadata, &v4.SigningError{
+					Err: fmt.Errorf("(GCSRestoreHeaders) unexpected request type %T", in.Request),
+				}
+			}
+
+			if saved, _ := middleware.GetStackValue(ctx, gcsIgnoredHeadersKey{}).(string); saved != "" {
+				req.Header.Set("Accept-Encoding", saved)
+			}
+
+			return next.HandleFinalize(ctx, in)
+		},
+	)
+}

backend/s3proxy/s3.go

@@ -58,6 +58,7 @@ type S3Proxy struct {
 	sslSkipVerify             bool
 	usePathStyle              bool
 	debug                     bool
+	gcsCompatibility          bool
 }
 
 var _ backend.Backend = &S3Proxy{}
@@ -70,7 +71,7 @@ func NewWithClient(ctx context.Context, client *s3.Client, metaBucket string) (*
 	return s, s.validate(ctx)
 }
 
-func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, anonymousCredentials, disableChecksum, disableDataIntegrityCheck, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
+func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, anonymousCredentials, disableChecksum, disableDataIntegrityCheck, sslSkipVerify, usePathStyle, debug, gcsCompatibility bool) (*S3Proxy, error) {
 	s := &S3Proxy{
 		access:                    access,
 		secret:                    secret,
@@ -83,6 +84,7 @@ func New(ctx context.Context, access, secret, endpoint, region, metaBucket strin
 		sslSkipVerify:             sslSkipVerify,
 		usePathStyle:              usePathStyle,
 		debug:                     debug,
+		gcsCompatibility:          gcsCompatibility,
 	}
 	client, err := s.getClientWithCtx(ctx)
 	if err != nil {

cmd/versitygw/s3.go

@@ -33,6 +33,7 @@ var (
 	s3proxySslSkipVerify             bool
 	s3proxyUsePathStyle              bool
 	s3proxyDebug                     bool
+	s3proxyGCSCompatibility          bool
 )
 
 func s3Command() *cli.Command {
@@ -121,13 +122,20 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
+			&cli.BoolFlag{
+				Name:        "gcs-compatibility",
+				Usage:       "enable GCS S3 compatibility mode",
+				Value:       false,
+				EnvVars:     []string{"VGW_S3_GCS_COMPATIBILITY"},
+				Destination: &s3proxyGCSCompatibility,
+			},
 		},
 	}
 }
 
 func runS3(ctx *cli.Context) error {
 	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyMetaBucket, s3proxyAnonymousCredentials, s3proxyDisableChecksum, s3proxyDisableDataIntegrityCheck, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
+		s3proxyMetaBucket, s3proxyAnonymousCredentials, s3proxyDisableChecksum, s3proxyDisableDataIntegrityCheck, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug, s3proxyGCSCompatibility)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

extra/example.conf

@@ -628,6 +628,14 @@ ROOT_SECRET_ACCESS_KEY=
 # VGW_S3_DEBUG will enable debug logging for S3 requests.
 #VGW_S3_DEBUG=false
 
+# VGW_S3_GCS_COMPATIBILITY enables Google Cloud Storage (GCS) S3 compatibility
+# mode. When enabled, the Accept-Encoding header is excluded from the SigV4
+# signed headers and restored after signing. This works around a signature
+# mismatch caused by the AWS SDK v2 including Accept-Encoding in signed headers,
+# which GCS does not support. See https://github.com/aws/aws-sdk-go-v2/issues/1816
+# for details.
+#VGW_S3_GCS_COMPATIBILITY=false
+
 ########
 # azure #
 ########