feat: add option to disable s3proxy client data integrity checks

AWS introduced a relatively newer option for data integrity checks
that not all non-AWS server support yet. See this for mmore info:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html

This change adds a new option: disable-data-integrity-check
to disable the data integrity checks in the client sdk for the
servers that may not yet support this. Use this only when the s3
service for the proxy does not support the data integrity features.

Fixes #1867

backend/s3proxy/client.go

@@ -76,6 +76,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
 	}
 
+	if s.disableDataIntegrityCheck {
+		opts = append(opts,
+			config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired))
+	}
+
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))

backend/s3proxy/s3.go

@@ -47,15 +47,16 @@ type S3Proxy struct {
 
 	client *s3.Client
 
-	access          string
-	secret          string
-	endpoint        string
-	awsRegion       string
-	metaBucket      string
-	disableChecksum bool
-	sslSkipVerify   bool
-	usePathStyle    bool
-	debug           bool
+	access                    string
+	secret                    string
+	endpoint                  string
+	awsRegion                 string
+	metaBucket                string
+	disableChecksum           bool
+	disableDataIntegrityCheck bool
+	sslSkipVerify             bool
+	usePathStyle              bool
+	debug                     bool
 }
 
 var _ backend.Backend = &S3Proxy{}
@@ -68,17 +69,18 @@ func NewWithClient(ctx context.Context, client *s3.Client, metaBucket string) (*
 	return s, s.validate(ctx)
 }
 
-func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
+func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, disableDataIntegrityCheck, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
-		access:          access,
-		secret:          secret,
-		endpoint:        endpoint,
-		awsRegion:       region,
-		metaBucket:      metaBucket,
-		disableChecksum: disableChecksum,
-		sslSkipVerify:   sslSkipVerify,
-		usePathStyle:    usePathStyle,
-		debug:           debug,
+		access:                    access,
+		secret:                    secret,
+		endpoint:                  endpoint,
+		awsRegion:                 region,
+		metaBucket:                metaBucket,
+		disableChecksum:           disableChecksum,
+		disableDataIntegrityCheck: disableDataIntegrityCheck,
+		sslSkipVerify:             sslSkipVerify,
+		usePathStyle:              usePathStyle,
+		debug:                     debug,
 	}
 	client, err := s.getClientWithCtx(ctx)
 	if err != nil {

cmd/versitygw/s3.go

@@ -22,15 +22,16 @@ import (
 )
 
 var (
-	s3proxyAccess          string
-	s3proxySecret          string
-	s3proxyEndpoint        string
-	s3proxyRegion          string
-	s3proxyMetaBucket      string
-	s3proxyDisableChecksum bool
-	s3proxySslSkipVerify   bool
-	s3proxyUsePathStyle    bool
-	s3proxyDebug           bool
+	s3proxyAccess                    string
+	s3proxySecret                    string
+	s3proxyEndpoint                  string
+	s3proxyRegion                    string
+	s3proxyMetaBucket                string
+	s3proxyDisableChecksum           bool
+	s3proxyDisableDataIntegrityCheck bool
+	s3proxySslSkipVerify             bool
+	s3proxyUsePathStyle              bool
+	s3proxyDebug                     bool
 )
 
 func s3Command() *cli.Command {
@@ -84,6 +85,13 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-data-integrity-check",
+				Usage:       "disable data integrity checks for requests (sets RequestChecksumCalculationWhenRequired)",
+				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_DATA_INTEGRITY_CHECK"},
+				Destination: &s3proxyDisableDataIntegrityCheck,
+			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
@@ -111,7 +119,7 @@ to an s3 storage backend service.`,
 
 func runS3(ctx *cli.Context) error {
 	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxyDisableDataIntegrityCheck, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

extra/example.conf

@@ -535,12 +535,23 @@ ROOT_SECRET_ACCESS_KEY=
 # be defined for an authorized access or both empty for an anonymous access.
 # The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will default to
 # "us-east-1" and "https://s3.amazonaws.com" respectively.
+# VGW_S3_META_BUCKET specifies a bucket to store bucket ACL/policy metadata.
+# VGW_S3_DISABLE_CHECKSUM will disable the SHA256 checksum for unsigned payload.
+# VGW_S3_DISABLE_DATA_INTEGRITY_CHECK will disable data integrity checks on
+# requests by setting RequestChecksumCalculationWhenRequired, which only
+# calculates checksums when explicitly required (may improve performance).
+# VGW_S3_SSL_SKIP_VERIFY will skip SSL certificate verification.
+# VGW_S3_USE_PATH_STYLE will use path style addressing for the S3 proxy.
+# VGW_S3_DEBUG will enable debug logging for S3 requests.
 #VGW_S3_ACCESS_KEY=
 #VGW_S3_SECRET_KEY=
 #VGW_S3_REGION=
 #VGW_S3_ENDPOINT=
+#VGW_S3_META_BUCKET=
 #VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_DISABLE_DATA_INTEGRITY_CHECK=false
 #VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_USE_PATH_STYLE=false
 #VGW_S3_DEBUG=false
 
 ########