Merge pull request #1884 from versity/ben/s3proxy-disable-data-integrity

feat: add option to disable s3proxy client data integrity checks

backend/s3proxy/client.go

@@ -76,6 +76,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
 			config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
 	}
 
+	if s.disableDataIntegrityCheck {
+		opts = append(opts,
+			config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired))
+	}
+
 	if s.debug {
 		opts = append(opts,
 			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))

backend/s3proxy/s3.go

@@ -47,15 +47,16 @@ type S3Proxy struct {
 
 	client *s3.Client
 
-	access          string
-	secret          string
-	endpoint        string
-	awsRegion       string
-	metaBucket      string
-	disableChecksum bool
-	sslSkipVerify   bool
-	usePathStyle    bool
-	debug           bool
+	access                    string
+	secret                    string
+	endpoint                  string
+	awsRegion                 string
+	metaBucket                string
+	disableChecksum           bool
+	disableDataIntegrityCheck bool
+	sslSkipVerify             bool
+	usePathStyle              bool
+	debug                     bool
 }
 
 var _ backend.Backend = &S3Proxy{}
@@ -68,17 +69,18 @@ func NewWithClient(ctx context.Context, client *s3.Client, metaBucket string) (*
 	return s, s.validate(ctx)
 }
 
-func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
+func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, disableDataIntegrityCheck, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
 	s := &S3Proxy{
-		access:          access,
-		secret:          secret,
-		endpoint:        endpoint,
-		awsRegion:       region,
-		metaBucket:      metaBucket,
-		disableChecksum: disableChecksum,
-		sslSkipVerify:   sslSkipVerify,
-		usePathStyle:    usePathStyle,
-		debug:           debug,
+		access:                    access,
+		secret:                    secret,
+		endpoint:                  endpoint,
+		awsRegion:                 region,
+		metaBucket:                metaBucket,
+		disableChecksum:           disableChecksum,
+		disableDataIntegrityCheck: disableDataIntegrityCheck,
+		sslSkipVerify:             sslSkipVerify,
+		usePathStyle:              usePathStyle,
+		debug:                     debug,
 	}
 	client, err := s.getClientWithCtx(ctx)
 	if err != nil {

cmd/versitygw/s3.go

@@ -22,15 +22,16 @@ import (
 )
 
 var (
-	s3proxyAccess          string
-	s3proxySecret          string
-	s3proxyEndpoint        string
-	s3proxyRegion          string
-	s3proxyMetaBucket      string
-	s3proxyDisableChecksum bool
-	s3proxySslSkipVerify   bool
-	s3proxyUsePathStyle    bool
-	s3proxyDebug           bool
+	s3proxyAccess                    string
+	s3proxySecret                    string
+	s3proxyEndpoint                  string
+	s3proxyRegion                    string
+	s3proxyMetaBucket                string
+	s3proxyDisableChecksum           bool
+	s3proxyDisableDataIntegrityCheck bool
+	s3proxySslSkipVerify             bool
+	s3proxyUsePathStyle              bool
+	s3proxyDebug                     bool
 )
 
 func s3Command() *cli.Command {
@@ -84,6 +85,13 @@ to an s3 storage backend service.`,
 				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-data-integrity-check",
+				Usage:       "disable data integrity checks for requests (sets RequestChecksumCalculationWhenRequired)",
+				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_DATA_INTEGRITY_CHECK"},
+				Destination: &s3proxyDisableDataIntegrityCheck,
+			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
@@ -111,7 +119,7 @@ to an s3 storage backend service.`,
 
 func runS3(ctx *cli.Context) error {
 	be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
-		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
+		s3proxyMetaBucket, s3proxyDisableChecksum, s3proxyDisableDataIntegrityCheck, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
 	if err != nil {
 		return fmt.Errorf("init s3 backend: %w", err)
 	}

extra/example.conf

@@ -553,12 +553,23 @@ ROOT_SECRET_ACCESS_KEY=
 # be defined for an authorized access or both empty for an anonymous access.
 # The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will default to
 # "us-east-1" and "https://s3.amazonaws.com" respectively.
+# VGW_S3_META_BUCKET specifies a bucket to store bucket ACL/policy metadata.
+# VGW_S3_DISABLE_CHECKSUM will disable the SHA256 checksum for unsigned payload.
+# VGW_S3_DISABLE_DATA_INTEGRITY_CHECK will disable data integrity checks on
+# requests by setting RequestChecksumCalculationWhenRequired, which only
+# calculates checksums when explicitly required (may improve performance).
+# VGW_S3_SSL_SKIP_VERIFY will skip SSL certificate verification.
+# VGW_S3_USE_PATH_STYLE will use path style addressing for the S3 proxy.
+# VGW_S3_DEBUG will enable debug logging for S3 requests.
 #VGW_S3_ACCESS_KEY=
 #VGW_S3_SECRET_KEY=
 #VGW_S3_REGION=
 #VGW_S3_ENDPOINT=
+#VGW_S3_META_BUCKET=
 #VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_DISABLE_DATA_INTEGRITY_CHECK=false
 #VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_USE_PATH_STYLE=false
 #VGW_S3_DEBUG=false
 
 ########