Merge pull request #464 from versity/ben/presign_escape

fix: escape path and query for presign signature validation

.github/workflows/system.yml

@@ -41,7 +41,6 @@ jobs:
       - name: Build and run
         run: |
           make testbin
-          echo $PATH
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
@@ -52,4 +51,9 @@ jobs:
           export WORKSPACE=$GITHUB_WORKSPACE
           openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
-          VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
+          mkdir /tmp/cover
+          VERSITYGW_TEST_ENV=./tests/.env.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
+
+      - name: Coverage report
+        run: |
+          go tool covdata percent -i=/tmp/cover

.gitignore

@@ -41,7 +41,7 @@ VERSION
 dist/
 
 # secrets file for local github-actions testing
-tests/.secrets
+tests/.secrets*
 
 # IAM users files often created in testing
 users.json

.goreleaser.yaml

@@ -32,11 +32,28 @@ archives:
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
+
+    # Set this to true if you want all files in the archive to be in a single directory.
+    # If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
+    # you'll get a folder 'goreleaser_Linux_arm64'.
+    # If set to false, all files are extracted separately.
+    # You can also set it to a custom folder name (templating is supported).
+    wrap_in_directory: true
+
     # use zip for windows archives
     format_overrides:
     - goos: windows
       format: zip
 
+    # Additional files/globs you want to add to the archive.
+    #
+    # Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
+    # Templates: allowed
+    files:
+      - README.md
+      - LICENSE
+      - NOTICE
+
 checksum:
   name_template: 'checksums.txt'
 
@@ -83,6 +100,27 @@ nfpms:
 
     rpm:
       group: "System Environment/Daemons"
+      # RPM specific scripts.
+      scripts:
+        # The pretrans script runs before all RPM package transactions / stages.
+        #pretrans: ./extra/pretrans.sh
+        # The posttrans script runs after all RPM package transactions / stages.
+        posttrans: ./extra/posttrans.sh
+
+    contents:
+      - src: extra/versitygw@.service
+        dst: /lib/systemd/system/versitygw@.service
+
+      - src: extra/example.conf
+        dst: /etc/versitygw.d/example.conf
+        type: config
+  
+      - dst: /etc/versitygw.d
+        type: dir
+        file_info:
+          mode: 0700
+
+
 
 # yaml-language-server: $schema=https://goreleaser.com/static/schema.json
 # vim: set ts=2 sw=2 tw=0 fo=cnqoj

Makefile

@@ -59,20 +59,13 @@ cleanall: clean
 	rm -f $(BIN)
 	rm -f versitygw-*.tar
 	rm -f versitygw-*.tar.gz
-	rm -f versitygw.spec
-
-%.spec: %.spec.in
-	sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
-	mv $@+ $@
 
 TARFILE = $(BIN)-$(VERSION).tar
 
-dist: $(BIN).spec
+dist:
 	echo $(VERSION) >VERSION
 	git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
-	@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
 	rm -f VERSION
-	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
 
 # Creates and runs S3 gateway instance in a docker container

backend/posix/posix.go

@@ -74,6 +74,12 @@ func New(rootdir string) (*Posix, error) {
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
+	_, err = xattr.FGet(f, "user.test")
+	if errors.Is(err, syscall.ENOTSUP) {
+		f.Close()
+		return nil, fmt.Errorf("xattr not supported on %v", rootdir)
+	}
+
 	return &Posix{rootfd: f, rootdir: rootdir}, nil
 }
 

cmd/versitygw/main.go

@@ -197,13 +197,13 @@ func initFlags() []cli.Flag {
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
-			EnvVars:     []string{"LOGFILE"},
+			EnvVars:     []string{"LOGFILE", "VGW_ACCESS_LOG"},
 			Destination: &accessLog,
 		},
 		&cli.StringFlag{
 			Name:        "log-webhook-url",
 			Usage:       "webhook url to send the audit logs",
-			EnvVars:     []string{"WEBHOOK"},
+			EnvVars:     []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
 			Destination: &logWebhookURL,
 		},
 		&cli.StringFlag{
@@ -369,6 +369,10 @@ func initFlags() []cli.Flag {
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
+	if rootUserAccess == "" || rootUserSecret == "" {
+		return fmt.Errorf("root user access and secret key must be provided")
+	}
+
 	app := fiber.New(fiber.Config{
 		AppName:           "versitygw",
 		ServerHeader:      "VERSITYGW",
@@ -492,7 +496,6 @@ Loop:
 	for {
 		select {
 		case <-ctx.Done():
-			err = ctx.Err()
 			break Loop
 		case err = <-c:
 			break Loop
@@ -512,12 +515,18 @@ Loop:
 
 	err = iam.Shutdown()
 	if err != nil {
+		if saveErr == nil {
+			saveErr = err
+		}
 		fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
 	}
 
 	if logger != nil {
 		err := logger.Shutdown()
 		if err != nil {
+			if saveErr == nil {
+				saveErr = err
+			}
 			fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
 		}
 	}

cmd/versitygw/s3.go

@@ -44,6 +44,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server access key id",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_ACCESS_KEY"},
 				Destination: &s3proxyAccess,
 				Aliases:     []string{"a"},
 			},
@@ -52,6 +53,7 @@ to an s3 storage backend service.`,
 				Usage:       "s3 proxy server secret access key",
 				Value:       "",
 				Required:    true,
+				EnvVars:     []string{"VGW_S3_SECRET_KEY"},
 				Destination: &s3proxySecret,
 				Aliases:     []string{"s"},
 			},
@@ -59,23 +61,27 @@ to an s3 storage backend service.`,
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
 				Value:       "",
+				EnvVars:     []string{"VGW_S3_ENDPOINT"},
 				Destination: &s3proxyEndpoint,
 			},
 			&cli.StringFlag{
 				Name:        "region",
 				Usage:       "s3 service region, default 'us-east-1' if not specified",
 				Value:       "us-east-1",
+				EnvVars:     []string{"VGW_S3_REGION"},
 				Destination: &s3proxyRegion,
 			},
 			&cli.BoolFlag{
 				Name:        "disable-checksum",
 				Usage:       "disable gateway to server object checksums",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DISABLE_CHECKSUM"},
 				Destination: &s3proxyDisableChecksum,
 			},
 			&cli.BoolFlag{
 				Name:        "ssl-skip-verify",
 				Usage:       "skip ssl cert verification for s3 service",
+				EnvVars:     []string{"VGW_S3_SSL_SKIP_VERIFY"},
 				Value:       false,
 				Destination: &s3proxySslSkipVerify,
 			},
@@ -83,6 +89,7 @@ to an s3 storage backend service.`,
 				Name:        "debug",
 				Usage:       "output extra debug tracing",
 				Value:       false,
+				EnvVars:     []string{"VGW_S3_DEBUG"},
 				Destination: &s3proxyDebug,
 			},
 		},

cmd/versitygw/scoutfs.go

@@ -48,6 +48,7 @@ move interfaces as well as support for tiered filesystems.`,
 				Name:        "glacier",
 				Usage:       "enable glacier emulation mode",
 				Aliases:     []string{"g"},
+				EnvVars:     []string{"VGW_SCOUTFS_GLACIER"},
 				Destination: &glacier,
 			},
 		},

extra/example.conf

@@ -0,0 +1,272 @@
+###################################
+# VersityGW systemd configuration #
+###################################
+
+# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
+# For example, if the service name is "mygateway", then the file should be
+# named /etc/versitygw.d/mygateway.conf.
+# The systemd template file /lib/systemd/system/versitygw@.service will
+# automatically load the configuration file for the service name.
+# To start the gateway, use the following command:
+# systemctl start versitygw@mygateway
+# To enable the gateway to start on boot, use the following command:
+# systemctl enable versitygw@mygateway
+# To stop the gateway, use the following command:
+# systemctl stop versitygw@mygateway
+
+# There can be multiple gateway services running on the same host. Each
+# gateway service must have a unique service name with a unique configuration
+# file in /etc/versitygw.d/. They must also listen on different ports and/or
+# interfaces using the VGW_PORT option.
+
+##############################
+# VersityGW Required Options #
+##############################
+
+# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
+# This defines the backend that the VGW will use for data access.
+VGW_BACKEND=posix
+
+# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
+# as the the top level directory for the gateway.
+# All sub directories of the top level directory are treated as buckets,
+# and all files/directories below the "bucket directory" are treated as
+# the objects. The object name is split on "/" separator to translate
+# to posix storage.
+# For example:
+# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+VGW_BACKEND_ARG=
+
+############################
+# VersityGW Global Options #
+############################
+
+# commented options are the default values
+
+# The following must be set, and do not have default values
+# The access and secret options will specify the root account credentials.
+# The root account is granted full authorization to all API requests after
+# authentication.
+ROOT_ACCESS_KEY_ID=
+ROOT_SECRET_ACCESS_KEY=
+
+# The following are optional, and have the default values as listed
+
+# The VGW_PORT option will specify the listening port for the S3 server.
+# This option can use either the form <ip>:<port> which will listen only
+# on the network interface that matches the IP on the specified port, or
+# :<port> which will listen on all network interfaces on the specified port.
+# The <ip> spec can either be IP dotted notation or a resolvable hostname.
+# The <port> spec can either be a numeric port or the service name typically
+# in /etc/services.
+#VGW_PORT=:7070
+
+# The VGW_REGION option will specify the region that the S3 server will
+# report to clients. This option is optional, and defaults to "us-east-1".
+#VGW_REGION=us-east-1
+
+# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
+# private key that the S3 server will use for SSL connections. This option
+# is optional, and defaults to not using SSL.
+#VGW_CERT=
+#VGW_KEY=
+
+# The VGW_ADMIN_PORT option will specify the listening port for the admin
+# server. The admin server endpoint can optionally be set to listen on a
+# different interface or port than the S3 service. This allows for better
+# control of firewall restrictions to the admin endpoint. The certs for this
+# can be different certs than specified for the S3 service. The default when
+# these are not specified is to have the admin server listen on the same
+# endpoint as the S3 service.
+# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
+# server will use SSL.
+#VGW_ADMIN_PORT=
+#VGW_ADMIN_CERT=
+#VGW_ADMIN_CERT_KEY=
+
+# The VGW_QUIET option when set will supress the S3 server request summary
+# logging to stdout.
+#VGW_QUIET=false
+
+# The VGW_HEALTH option when set will specify the URL to accept health checks
+# on. The health check endpoint is often used for load balancers to verify
+# gateway is alive. The health endpoint masks any bucket with this setting.
+# For example, if the health endpoint is set to /health, the gateway will not
+# allow creating or listing contents of a bucket called "health". The health
+# endpoint is unauthenticated, and returns a 200 status for GET.
+#VGW_HEALTH=
+
+###############
+# Access Logs #
+###############
+
+# The VGW_ACCESS_LOG option when set will specify the file to log all S3
+# server requests to. This option is optional, and defaults to not logging.
+# It is suggested to use absolute paths for the server log file because the
+# server may chdir into the backend root directory and change locations for
+# relative paths.
+# The log file format follows the AWS S3 access log format documented in
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
+#VGW_ACCESS_LOG=
+
+# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
+# S3 server request access logs to. The access logs are JSON encoded when
+# sent to the webhook.
+#VGW_LOG_WEBHOOK_URL=
+
+##############
+# Event Logs #
+##############
+
+# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
+# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
+# bucket events will be sent to the kafka service. The gateway events are
+# similar to AWS S3 events, and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_KAFKA_URL=
+#VGW_EVENT_KAFKA_TOPIC=
+#VGW_EVENT_KAFKA_KEY=
+
+# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
+# and VGW_EVENT_NATS_TOPIC are specified, all bucket events will be sent to the
+# the NATS messaging service. The gateway events are  similar to AWS S3 events,
+# and are documented in the wiki:
+# https://github.com/versity/versitygw/wiki/Events-Notifications.
+#VGW_EVENT_NATS_URL=
+#VGW_EVENT_NATS_TOPIC=
+
+# The VGW_DEBUG option enables verbose debug log output to stdout. This output
+# includes details for signature verification steps. This is generally only
+# useful for debugging the S3 server, and should not be used in production.
+#VGW_DEBUG=false
+
+################
+# IAM services #
+################
+
+# The VGW_IAM_DIR option will enable the internal IAM service with accounts
+# stored in a file under the specified directory. This is provided to minimize
+# dependencies on outside services for basic functionality. The local account
+# files are plain text and only protected with file permissions. This IAM
+# service is added for convenience, but is not considered as secure or scalable
+# as a dedicated IAM service.
+#VGW_IAM_DIR=
+
+# The ldap options will enable the LDAP IAM service with accounts stored in an
+# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
+# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
+# secret credentials and role respectively. The other options are used to
+# connect to the LDAP service.
+#VGW_IAM_LDAP_URL=
+#VGW_IAM_LDAP_BASE_DN=
+#VGW_IAM_LDAP_BIND_DN=
+#VGW_IAM_LDAP_BIND_PASS=
+#VGW_IAM_LDAP_QUERY_BASE=
+#VGW_IAM_LDAP_OBJECT_CLASSES=
+#VGW_IAM_LDAP_ACCESS_ATR=
+#VGW_IAM_LDAP_SECRET_ATR=
+#VGW_IAM_LDAP_ROLE_ATR=
+
+# The VGW_S3 IAM service is similar to the internal IAM service, but instead
+# stores the account information JSON encoded in an S3 object. This should use
+# a bucket that is not accessible to general users when using s3 backend to
+# prevent access to account credentials. This IAM service is added for
+# convenience, but is not considered as secure or scalable as a dedicated IAM
+# service.
+#VGW_S3_IAM_ACCESS_KEY=
+#VGW_S3_IAM_SECRET_KEY=
+#VGW_S3_IAM_REGION=
+#VGW_S3_IAM_ENDPOINT=
+#VGW_S3_IAM_BUCKET=
+#VGW_S3_IAM_NO_VERIFY=
+
+###############
+# IAM caching #
+###############
+
+# The IAM cache is intended to ease the load on the IAM service and increase
+# the Gateway performance by caching accounts and credentials for the TTL time
+# interval. Disabling this will cause a request to the configured IAM service
+# for each incoming request to retrieve the corresponding account credentials.
+# The cache is enabled by default. The TTL specifies how long to cache
+# credentials, and the prune value determines the interval for expired entries
+# to be removed from the cache. Increasing the TTL may lessen the load on the
+# IAM service backend, but may have out of date account info until the next
+# interval. Increasing the prune value may reduce memory use at the cost of
+# added CPU to check cache expirations.
+#VGW_IAM_CACHE_DISABLE=false
+#VGW_IAM_CACHE_TTL=120
+#VGW_IAM_CACHE_PRUNE=3600
+
+######################################
+# VersityGW Backend Specific Options #
+######################################
+
+#########
+# posix #
+#########
+
+# The posix backend translates S3 requests to file access in a local filesystem.
+# The posix backend requires a filesystem that supports extended attributes.
+# The top level directory for the gateway must be provided. All sub directories
+# of the top level directory are treated as buckets, and all files/directories
+# below the "bucket directory" are treated as the objects. The object
+# name is split on "/" separator to translate to posix storage.
+# For example:
+# top level: /mnt/fs/gwroot
+# bucket: mybucket
+# object: a/b/c/myobject
+# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
+
+# There are currently no further options other than VGW_BACKEND_ARG for the
+# posix backend.
+
+###########
+# scoutfs #
+###########
+
+# The scoutfs backend requires a ScoutFS filesystem type for the backend
+# path. The glacier mode functionality requires ScoutAM to be configured
+# for tiering data from the ScoutFS filesystem to a mass stroage system.
+# The mass storage system is often one or more tape libraries. Due to the
+# high latency of tape, the glacier mode functionality is designed to
+# give feedback to clients about object state and offer ability to request
+# data to be staged back to disk without the client dealing with long
+# request timeout settings.
+
+# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
+# GET object:  if file offline, return invalid object state
+# HEAD object: if file offline, set obj storage class to GLACIER
+#              if file offline and staging, x-amz-restore: ongoing-request="true"
+#              if file offline and not staging, x-amz-restore: ongoing-request="false"
+#              if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
+#              note: this expiry-date is not used but provided for client glacier compatibility
+# ListObjects: if file offline, set obj storage class to GLACIER
+# RestoreObject: add batch stage request to file
+#VGW_SCOUTFS_GLACIER=false
+
+######
+# s3 #
+######
+
+# The s3 backend allows the gateway to forward requests to an S3 compatible
+# service. This allows the gateway to act as a proxy for another S3 service.
+# The backend S3 access is all done with a single configured account. The
+# gateway will manage incoming multi-tenant access with the gateway configured
+# IAM service. This gives stroage admins the ability to manage local gateway
+# accounts while maintaining full control and a single account for the backend
+# S3 service.
+
+# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
+# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
+# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
+#VGW_S3_ACCESS_KEY=
+#VGW_S3_SECRET_KEY=
+#VGW_S3_REGION=
+#VGW_S3_ENDPOINT=
+#VGW_S3_DISABLE_CHECKSUM=false
+#VGW_S3_SSL_SKIP_VERIFY=false
+#VGW_S3_DEBUG=false

extra/posttrans.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+systemctl daemon-reload

extra/versitygw@.service

@@ -0,0 +1,33 @@
+[Unit]
+Description=VersityGW
+Documentation=https://github.com/versity/versitygw/wiki
+Wants=network-online.target
+After=network-online.target remote-fs.target
+AssertFileIsExecutable=/usr/bin/versitygw
+AssertPathExists=/etc/versitygw.d/%i.conf
+
+[Service]
+WorkingDirectory=/root
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=versitygw-%i
+
+User=root
+Group=root
+
+EnvironmentFile=/etc/versitygw.d/%i.conf
+
+ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
+
+# Let systemd restart this service always
+Restart=always
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Specifies the maximum number of threads this process can create
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
+

go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
-	github.com/aws/aws-sdk-go-v2 v1.25.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4
+	github.com/aws/aws-sdk-go-v2 v1.26.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
 	github.com/aws/smithy-go v1.20.1
 	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/gofiber/fiber/v2 v2.52.2
@@ -26,11 +26,11 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -47,16 +47,16 @@ require (
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.7
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.7
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.8
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.8
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -16,42 +16,42 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.25.3 h1:xYiLpZTQs1mzvz5PaI6uR0Wh57ippuEthxS4iK5v0n0=
-github.com/aws/aws-sdk-go-v2 v1.25.3/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
+github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA=
+github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
-github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4=
-github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 h1:vXY/Hq1XdxHBIYgBUmug/AbMyIe1AKulPYS2/VE1X70=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9/go.mod h1:GyJJTZoHVuENM4TeJEl5Ffs4W9m19u+4wKJcDi/GZ4A=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw=
+github.com/aws/aws-sdk-go-v2/config v1.27.8 h1:0r8epOsiJ7YJz65MGcb8i91ehFp4kvvFe2qkq5oYeRI=
+github.com/aws/aws-sdk-go-v2/config v1.27.8/go.mod h1:XsmYKxYNuIhLsFddpNds+j9H5XKzjWDdg/SZngiwFio=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8 h1:WUdNLXbyNbU07V/WFrSOBXqZTDgmmMNMgUFzpYOKJhw=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.8/go.mod h1:iPZzLpaBIfhyvVS/XGD3JvR1GP3YdHTqpySKDlqkfs8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4 h1:S+L2QSKhUuShih3aq9P/mkzDBiOO5tTyVg+vXREfsfg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.4/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12 h1:rfAytUY7OgbOMDkzxdiigZkbTe9SDER2dIpO/Fzi9+0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.12/go.mod h1:BaY3WWSgUwV/zq0K3HePyXhRYZxGnDATYERkR0f1RTs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 h1:mDnFOE2sVkyphMWtTH+stv0eW3k0OTx94K63xpxHty4=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3/go.mod h1:V8MuRVcCRt5h1S+Fwu8KbC7l/gBGo3yBAyUbJM2IJOk=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 h1:mbWNpfRUTT6bnacmvOTKXZjR/HycibdWzNpfbrbLDIs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5/go.mod h1:FCOPWGjsshkkICJIn9hq9xr6dLKtyaWpuUojiN3W1/8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 h1:4t+QEX7BsXz98W8W1lNvMAG+NX8qHz2CjLBxQKku40g=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3/go.mod h1:oFcjjUq5Hm09N9rpxTdeMeLeQcxS7mIkBkL8qUKng+A=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4 h1:lW5xUzOPGAMY7HPuNF4FdyBwRc3UJ/e8KsapbesVeNU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4/go.mod h1:MGTaf3x/+z7ZGugCGvepnx2DS6+caCYYqKhzVoLNYPk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
 github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=

s3api/utils/auth_test.go

@@ -84,7 +84,7 @@ func Test_Client_UserAgent(t *testing.T) {
 		}
 
 		req.Host = host
-		req.Header.Add("X-Amz-Content-Sha256", zeroLenSig)
+		req.Header.Set("X-Amz-Content-Sha256", zeroLenSig)
 
 		signer := v4.NewSigner()
 

s3api/utils/utils.go

@@ -20,11 +20,13 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/aws/smithy-go/encoding/httpbinding"
 	"github.com/gofiber/fiber/v2"
 	"github.com/valyala/fasthttp"
 	"github.com/versity/versitygw/s3err"
@@ -73,6 +75,13 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
 		}
 	})
 
+	// make sure all headers in the signed headers are present
+	for _, header := range signedHdrs {
+		if httpReq.Header.Get(header) == "" {
+			httpReq.Header.Set(header, "")
+		}
+	}
+
 	// Check if Content-Length in signed headers
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
@@ -107,16 +116,18 @@ func createPresignedHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, cont
 	}
 
 	uri := string(ctx.Request().URI().Path())
+	uri = httpbinding.EscapePath(uri, false)
 	isFirst := true
 
 	ctx.Request().URI().QueryArgs().VisitAll(func(key, value []byte) {
 		_, ok := signedQueryArgs[string(key)]
 		if !ok {
+			escapeValue := url.QueryEscape(string(value))
 			if isFirst {
-				uri += fmt.Sprintf("?%s=%s", key, value)
+				uri += fmt.Sprintf("?%s=%s", key, escapeValue)
 				isFirst = false
 			} else {
-				uri += fmt.Sprintf("&%s=%s", key, value)
+				uri += fmt.Sprintf("&%s=%s", key, escapeValue)
 			}
 		}
 	})

s3api/utils/utils_test.go

@@ -35,6 +35,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 		args    args
 		want    *http.Request
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -43,6 +44,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request,
 			wantErr: false,
+			hdrs:    []string{},
 		},
 		{
 			name: "Success-response-With-Headers",
@@ -51,11 +53,12 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 			},
 			want:    request2,
 			wantErr: false,
+			hdrs:    []string{"X-Amz-Mfa"},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
+			got, err := createHttpRequestFromCtx(tt.args.ctx, tt.hdrs, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return

tests/.env.default

@@ -1,6 +1,7 @@
 AWS_PROFILE=versity
 AWS_ENDPOINT_URL=https://127.0.0.1:7070
 VERSITY_EXE=./versitygw
+RUN_VERSITYGW=true
 BACKEND=posix
 LOCAL_FOLDER=/tmp/gw
 BUCKET_ONE_NAME=versity-gwtest-bucket-one
@@ -8,4 +9,6 @@ BUCKET_TWO_NAME=versity-gwtest-bucket-two
 #RECREATE_BUCKETS=true
 CERT=$PWD/cert.pem
 KEY=$PWD/versitygw.pem
-S3CMD_CONFIG=./tests/s3cfg.local.default
+S3CMD_CONFIG=./tests/s3cfg.local.default
+SECRETS_FILE=./tests/.secrets
+MC_ALIAS=versity

tests/.env.static

@@ -1,8 +0,0 @@
-AWS_PROFILE=versity
-AWS_ENDPOINT_URL=http://127.0.0.1:7070
-VERSITY_EXE=./versitygw
-BACKEND=posix
-LOCAL_FOLDER=/tmp/gw
-BUCKET_ONE_NAME=versity-gwtest-bucket-one-static
-BUCKET_TWO_NAME=versity-gwtest-bucket-two-static
-RECREATE_BUCKETS=false

tests/integration/group-tests.go

@@ -43,6 +43,7 @@ func TestPresignedAuthentication(s *S3Conf) {
 	PresignedAuth_incorrect_secret_key(s)
 	PresignedAuth_PutObject_success(s)
 	PresignedAuth_Put_GetObject_with_data(s)
+	PresignedAuth_Put_GetObject_with_UTF8_chars(s)
 	PresignedAuth_UploadPart(s)
 }
 
@@ -329,6 +330,7 @@ func GetIntTests() IntTests {
 		"PresignedAuth_incorrect_secret_key":                    PresignedAuth_incorrect_secret_key,
 		"PresignedAuth_PutObject_success":                       PresignedAuth_PutObject_success,
 		"PresignedAuth_Put_GetObject_with_data":                 PresignedAuth_Put_GetObject_with_data,
+		"PresignedAuth_Put_GetObject_with_UTF8_chars":           PresignedAuth_Put_GetObject_with_UTF8_chars,
 		"PresignedAuth_UploadPart":                              PresignedAuth_UploadPart,
 		"CreateBucket_invalid_bucket_name":                      CreateBucket_invalid_bucket_name,
 		"CreateBucket_existing_bucket":                          CreateBucket_existing_bucket,

tests/integration/tests.go

@@ -1448,8 +1448,6 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 			return fmt.Errorf("read get object response body %w", err)
 		}
 
-		fmt.Println(resp.Request.Method, resp.ContentLength, string(respBody))
-
 		if string(respBody) != data {
 			return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody)
 		}
@@ -1463,6 +1461,72 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error {
 	})
 }
 
+func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
+	testName := "PresignedAuth_Put_GetObject_with_data"
+	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {
+		bucket, obj := getBucketName(), "my-$%^&*;"
+		err := setup(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		httpClient := http.Client{
+			Timeout: shortTimeout,
+		}
+
+		req, err := http.NewRequest(v4req.Method, v4req.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		req.Header = v4req.SignedHeader
+
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected my-obj to be successfully uploaded and get %v response status, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
+		v4GetReq, err := client.PresignGetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &obj})
+		cancel()
+		if err != nil {
+			return err
+		}
+
+		req, err = http.NewRequest(v4GetReq.Method, v4GetReq.URL, nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err = httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode)
+		}
+
+		err = teardown(s, bucket)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func PresignedAuth_UploadPart(s *S3Conf) error {
 	testName := "PresignedAuth_UploadPart"
 	return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error {

tests/setup.sh

@@ -17,22 +17,25 @@ setup() {
     return 1
   fi
 
-  S3CMD_OPTS=()
-  S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
-  S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
-  S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+  if [[ $RUN_S3CMD == true ]]; then
+    S3CMD_OPTS=()
+    S3CMD_OPTS+=(-c "$S3CMD_CONFIG")
+    S3CMD_OPTS+=(--access_key="$AWS_ACCESS_KEY_ID")
+    S3CMD_OPTS+=(--secret_key="$AWS_SECRET_ACCESS_KEY")
+    export S3CMD_CONFIG S3CMD_OPTS
+  fi
 
-  check_add_mc_alias || check_result=$?
-  if [[ $check_result -ne 0 ]]; then
-    echo "mc alias check/add failed"
-    return 1
+  if [[ $RUN_MC == true ]]; then
+    check_add_mc_alias || check_result=$?
+    if [[ $check_result -ne 0 ]]; then
+      echo "mc alias check/add failed"
+      return 1
+    fi
   fi
 
   export AWS_PROFILE \
     BUCKET_ONE_NAME \
-    BUCKET_TWO_NAME \
-    S3CMD_CONFIG \
-    S3CMD_OPTS
+    BUCKET_TWO_NAME
 }
 
 # make sure required environment variables for tests are defined properly

tests/setup_mc.sh

@@ -8,7 +8,8 @@ check_for_alias() {
     return 2
   fi
   while IFS= read -r line; do
-    if [[ $line =~ ^versity$ ]]; then
+    error=$(echo "$line" | grep -w "$MC_ALIAS ")
+    if [[ $? -eq 0 ]]; then
       return 0
     fi
   done <<< "$aliases"
@@ -25,7 +26,7 @@ check_add_mc_alias() {
     return 0
   fi
   local set_result
-  error=$(mc alias set --insecure versity "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
+  error=$(mc alias set --insecure "$MC_ALIAS" "$AWS_ENDPOINT_URL" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY") || set_result=$?
   if [[ $set_result -ne 0 ]]; then
     echo "error setting alias: $error"
     return 1

tests/test_aws.sh

@@ -11,8 +11,12 @@ source ./tests/test_common.sh
 }
 
 # test adding and removing an object on versitygw
-@test "test_put_object" {
-  test_common_put_object "aws"
+@test "test_put_object-with-data" {
+  test_common_put_object_with_data "aws"
+}
+
+@test "test_put_object-no-data" {
+  test_common_put_object_no_data "aws"
 }
 
 # test listing buckets on versitygw
@@ -97,27 +101,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve bucket tags
 @test "test-set-get-bucket-tags" {
-
-  local key="test_key"
-  local value="test_value"
-
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_bucket_tag "$BUCKET_ONE_NAME" $key $value
-  get_bucket_tags "$BUCKET_ONE_NAME" || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  test_common_set_get_bucket_tags "aws"
 }
 
 # test v1 s3api list objects command
@@ -184,34 +168,7 @@ source ./tests/test_common.sh
 
 # test abilty to set and retrieve object tags
 @test "test-set-get-object-tags" {
-
-  local bucket_file="bucket-file"
-  local key="test_key"
-  local value="test_value"
-
-  create_test_files "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test files"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-  local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
-  put_object "aws" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
-  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
-
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
-  [[ $get_result -eq 0 ]] || fail "Error getting object tags"
-  tag_set=$(echo "$tags" | jq '.TagSet')
-  [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
-
-  put_object_tag "$BUCKET_ONE_NAME" $bucket_file $key $value
-  get_object_tags "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
-  [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
-  tag_set_key=$(echo "$tags" | jq '.TagSet[0].Key')
-  tag_set_value=$(echo "$tags" | jq '.TagSet[0].Value')
-  [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
-  [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
-
-  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
+  test_common_set_get_object_tags "aws"
 }
 
 # test multi-part upload

tests/test_common.sh

@@ -25,24 +25,38 @@ test_common_create_delete_bucket() {
   [[ $delete_result_two -eq 0 ]] || fail "Failed to delete bucket"
 }
 
-test_common_put_object() {
-
+test_common_put_object_with_data() {
   if [[ $# -ne 1 ]]; then
     fail "put object test requires command type"
   fi
 
   local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  echo "test data" > "$test_file_folder"/"$object_name"
+}
+
+test_common_put_object_no_data() {
+  if [[ $# -ne 1 ]]; then
+    fail "put object test requires command type"
+  fi
+
+  local object_name="test-object"
+  create_test_files "$object_name" || local create_result=$?
+  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+  test_common_put_object "$1" "$object_name"
+}
+
+test_common_put_object() {
+  if [[ $# -ne 2 ]]; then
+    fail "put object test requires command type, file"
+  fi
 
   setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
   [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
 
-  create_test_files "$object_name" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Error creating test file"
-
-  echo "test data" > "$test_file_folder"/"$object_name"
-
-  object="$BUCKET_ONE_NAME"/$object_name
-  put_object "$1" "$test_file_folder"/"$object_name" "$object" || local put_object=$?
+  object="$BUCKET_ONE_NAME"/"$2"
+  put_object "$1" "$test_file_folder"/"$2" "$object" || local put_object=$?
   [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
   object_exists "$1" "$object" || local exists_result_one=$?
   [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
@@ -53,7 +67,7 @@ test_common_put_object() {
   [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
-  delete_test_files "$object_name"
+  delete_test_files "$2"
 }
 
 # common test for listing buckets
@@ -95,7 +109,6 @@ test_common_list_buckets() {
 }
 
 test_common_list_objects() {
-
   if [[ $# -ne 1 ]]; then
     echo "common test function for listing objects requires command type"
     return 1
@@ -133,3 +146,113 @@ test_common_list_objects() {
     fail "$object_one and/or $object_two not listed (all objects: ${object_array[*]})"
   fi
 }
+
+test_common_set_get_bucket_tags() {
+
+  if [[ $# -ne 1 ]]; then
+    fail "set/get bucket tags test requires command type"
+  fi
+
+  local key="test_key"
+  local value="test_value"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "Error getting bucket tags"
+
+  if [[ $1 == 'aws' ]]; then
+    if [[ $tags != "" ]]; then
+      tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+      [[ $tag_set == "[]" ]] || fail "Error:  tags not empty: $tags"
+    fi
+  else
+    [[ $tags == "" ]] || [[ $tags =~ "No tags found" ]] || fail "Error:  tags not empty: $tags"
+  fi
+
+  put_bucket_tag "$1" "$BUCKET_ONE_NAME" $key $value
+  get_bucket_tags "$1" "$BUCKET_ONE_NAME" || local get_result_two=$?
+  [[ $get_result_two -eq 0 ]] || fail "Error getting bucket tags"
+
+  local tag_set_key
+  local tag_set_value
+  if [[ $1 == 'aws' ]]; then
+    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
+    [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
+  else
+    read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')"
+    [[ $tag_set_key == "$key" ]] || fail "Key mismatch"
+    [[ $tag_set_value == "$value" ]] || fail "Value mismatch"
+  fi
+  delete_bucket_tags "$1" "$BUCKET_ONE_NAME"
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+}
+
+test_common_set_get_object_tags() {
+
+  if [[ $# -ne 1 ]]; then
+    echo "get/set object tags missing command type"
+    return 1
+  fi
+
+  local bucket_file="bucket-file"
+  local key="test_key"
+  local value="test_value"
+
+  create_test_files "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test files"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  local object_path="$BUCKET_ONE_NAME"/"$bucket_file"
+  put_object "$1" "$test_file_folder"/"$bucket_file" "$object_path" || local put_object=$?
+  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket '$BUCKET_ONE_NAME'"
+
+  get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result=$?
+  [[ $get_result -eq 0 ]] || fail "Error getting object tags"
+  if [[ $1 == 'aws' ]]; then
+    tag_set=$(echo "$tags" | sed '1d' | jq '.TagSet')
+    [[ $tag_set == "[]" ]] || fail "Error:  tags not empty"
+  elif [[ ! $tags == *"No tags found"* ]]; then
+    fail "no tags found (tags: $tags)"
+  fi
+
+  put_object_tag "$1" "$BUCKET_ONE_NAME" $bucket_file $key $value
+  get_object_tags "$1" "$BUCKET_ONE_NAME" $bucket_file || local get_result_two=$?
+  [[ $get_result_two -eq 0 ]] || fail "Error getting object tags"
+  if [[ $1 == 'aws' ]]; then
+    tag_set_key=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Key')
+    tag_set_value=$(echo "$tags" | sed '1d' | jq '.TagSet[0].Value')
+    [[ $tag_set_key == '"'$key'"' ]] || fail "Key mismatch"
+    [[ $tag_set_value == '"'$value'"' ]] || fail "Value mismatch"
+  else
+    read -r tag_set_key tag_set_value <<< "$(echo "$tags" | awk 'NR==2 {print $1, $3}')"
+    [[ $tag_set_key == "$key" ]] || fail "Key mismatch"
+    [[ $tag_set_value == "$value" ]] || fail "Value mismatch"
+  fi
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}
+
+test_common_multipart_upload() {
+  if [[ $# -ne 1 ]]; then
+    echo "multipart upload command missing command type"
+    return 1
+  fi
+  bucket_file="largefile"
+
+  create_large_file "$bucket_file" || local created=$?
+  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
+
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
+  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+
+  put_object "$1" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
+  [[ $put_result -eq 0 ]] || fail "failed to copy file"
+
+  delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
+  delete_test_files $bucket_file
+}

tests/test_mc.sh

@@ -3,7 +3,37 @@
 source ./tests/test_common.sh
 source ./tests/setup.sh
 
+export RUN_MC=true
+
 # test mc bucket creation/deletion
 @test "test_create_delete_bucket_mc" {
   test_common_create_delete_bucket "mc"
-}
+}
+
+@test "test_put_object-with-data-mc" {
+  test_common_put_object_with_data "mc"
+}
+
+@test "test_put_object-no-data-mc" {
+  test_common_put_object_no_data "mc"
+}
+
+@test "test_list_buckets_mc" {
+  test_common_list_buckets "mc"
+}
+
+@test "test_list_objects_mc" {
+  test_common_list_objects "mc"
+}
+
+@test "test_set_get_bucket_tags_mc" {
+  test_common_set_get_bucket_tags "mc"
+}
+
+@test "test_set_get_object_tags_mc" {
+  test_common_set_get_object_tags "mc"
+}
+
+@test "test_multipart_upload_mc" {
+  test_common_multipart_upload "mc"
+}

tests/test_s3cmd.sh

@@ -4,14 +4,20 @@ source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/util.sh
 
+export RUN_S3CMD=true
+
 # test s3cmd bucket creation/deletion
 @test "test_create_delete_bucket_s3cmd" {
   test_common_create_delete_bucket "s3cmd"
 }
 
 # test s3cmd put object
-@test "test_put_object_s3cmd" {
-  test_common_put_object "s3cmd"
+@test "test_put_object_with_data_s3cmd" {
+  test_common_put_object_with_data "s3cmd"
+}
+
+@test "test_put_object_no_data_s3cmd" {
+  test_common_put_object_no_data "s3cmd"
 }
 
 # test listing buckets on versitygw
@@ -24,18 +30,5 @@ source ./tests/util.sh
 }
 
 @test "test_multipart_upload_s3cmd" {
-
-  bucket_file="largefile"
-
-  create_large_file "$bucket_file" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating test file for multipart upload"
-
-  setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
-
-  put_object "s3cmd" "$test_file_folder"/$bucket_file "$BUCKET_ONE_NAME/$bucket_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "failed to copy file"
-
-  delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
-  delete_test_files $bucket_file
+  test_common_multipart_upload "s3cmd"
 }

tests/util.sh

@@ -18,7 +18,7 @@ create_bucket() {
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate mb s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure mb versity/"$2" 2>&1) || exit_code=$?
+    error=$(mc --insecure mb "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -34,14 +34,21 @@ create_bucket() {
 # param:  bucket name
 # return 0 for success, 1 for failure
 delete_bucket() {
-  if [ $# -ne 1 ]; then
-    echo "delete bucket missing bucket name"
+  if [ $# -ne 2 ]; then
+    echo "delete bucket missing command type, bucket name"
     return 1
   fi
 
   local exit_code=0
   local error
-  error=$(aws --no-verify-ssl s3 rb s3://"$1" 2>&1) || exit_code="$?"
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3 rb s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure rb "$MC_ALIAS/$2" 2>&1) || exit_code=$?
+  else
+    echo "Invalid command type $1"
+    return 1
+  fi
   if [ $exit_code -ne 0 ]; then
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
@@ -79,7 +86,7 @@ delete_bucket_recursive() {
     if [[ "$error" == *"The specified bucket does not exist"* ]]; then
       return 0
     else
-      echo "error deleting bucket: $error"
+      echo "error deleting bucket recursively: $error"
       return 1
     fi
   fi
@@ -91,7 +98,7 @@ delete_bucket_recursive() {
 # return 0 for success, 1 for failure
 delete_bucket_contents() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket missing bucket name"
+    echo "delete bucket missing command id, bucket name"
     return 1
   fi
 
@@ -102,7 +109,7 @@ delete_bucket_contents() {
   elif [[ $1 == "s3cmd" ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate del s3://"$2" --recursive --force 2>&1) || exit_code="$?"
   elif [[ $1 == "mc" ]]; then
-    error=$(mc --insecure rm --force --recursive versity/"$2" 2>&1) || exit_code="$?"
+    error=$(mc --insecure rm --force --recursive "$MC_ALIAS"/"$2" 2>&1) || exit_code="$?"
   else
     echo "invalid command type $1"
     return 1
@@ -126,18 +133,20 @@ bucket_exists() {
   local exit_code=0
   local error
   if [[ $1 == 'aws' ]]; then
-    error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
+    error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
+    # NOTE:  s3cmd sometimes takes longer with direct connection
+    sleep 1
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
-    error=$(mc --insecure ls versity)
+    error=$(mc --insecure ls "$MC_ALIAS/$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type: $1"
     return 2
   fi
 
   if [ $exit_code -ne 0 ]; then
-    if [[ "$error" == *"The specified bucket does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
+    if [[ "$error" == *"does not exist"* ]] || [[ "$error" == *"Access Denied"* ]]; then
       return 1
     else
       echo "error checking if bucket exists: $error"
@@ -193,6 +202,7 @@ setup_bucket() {
       return 1
     fi
     if [[ $RECREATE_BUCKETS == "false" ]]; then
+      echo "bucket data deletion success"
       return 0
     fi
   fi
@@ -219,11 +229,13 @@ object_exists() {
     return 2
   fi
   local exit_code=0
-  local error
+  local error=""
   if [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code="$?"
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code="$?"
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 2
@@ -235,8 +247,8 @@ object_exists() {
       echo "error checking if object exists: $error"
       return 2
     fi
-  # s3cmd returns empty when object doesn't exist, rather than error
-  elif [[ $1 == 's3cmd' ]] && [[ $error == "" ]]; then
+  # s3cmd, mc return empty when object doesn't exist, rather than error
+  elif [[ ( $1 == 's3cmd' ) || ( $1 == 'mc' ) ]] && [[ $error == "" ]]; then
     return 1
   fi
   return 0
@@ -255,8 +267,9 @@ put_object() {
   if [[ $1 == 'aws' ]]; then
     error=$(aws --no-verify-ssl s3 cp "$2" s3://"$3" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
-    echo "2: $2 3: $(dirname $3)"
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$(dirname "$3")" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure cp "$2" "$MC_ALIAS"/"$(dirname "$3")" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -305,7 +318,8 @@ delete_object() {
     error=$(aws --no-verify-ssl s3 rm s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm s3://"$2" 2>&1) || exit_code=$?
-    echo "$error"
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure rm "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -322,7 +336,7 @@ delete_object() {
 # export bucket_array (bucket names) on success, return 1 for failure
 list_buckets() {
   if [[ $# -ne 1 ]]; then
-    echo "List buckets command mssing format"
+    echo "List buckets command missing format"
     return 1
   fi
 
@@ -331,7 +345,9 @@ list_buckets() {
   if [[ $1 == "aws" ]]; then
     output=$(aws --no-verify-ssl s3 ls s3:// 2>&1) || exit_code=$?
   elif [[ $1 == "s3cmd" ]]; then
-    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://) || exit_code=$?
+    output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3:// 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    output=$(mc --insecure ls "$MC_ALIAS" 2>&1) || exit_code=$?
   else
     echo "invalid format:  $1"
     return 1
@@ -345,7 +361,7 @@ list_buckets() {
   bucket_array=()
   while IFS= read -r line; do
     bucket_name=$(echo "$line" | awk '{print $NF}')
-    bucket_array+=("$bucket_name")
+    bucket_array+=("${bucket_name%/}")
   done <<< "$output"
 
   export bucket_array
@@ -365,6 +381,8 @@ list_objects() {
     output=$(aws --no-verify-ssl s3 ls s3://"$2" 2>&1) || exit_code=$?
   elif [[ $1 == 's3cmd' ]]; then
     output=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate ls s3://"$2" 2>&1) || exit_code=$?
+  elif [[ $1 == 'mc' ]]; then
+    output=$(mc --insecure ls "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
   else
     echo "invalid command type $1"
     return 1
@@ -463,13 +481,20 @@ get_object_acl() {
 # params:  bucket, key, value
 # return:  0 for success, 1 for error
 put_bucket_tag() {
-  if [ $# -ne 3 ]; then
-    echo "bucket tag command missing bucket name, key, value"
+  if [ $# -ne 4 ]; then
+    echo "bucket tag command missing command type, bucket name, key, value"
     return 1
   fi
   local error
   local result
-  error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$1" --tagging "TagSet=[{Key=$2,Value=$3}]") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-bucket-tagging --bucket "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2" "$3=$4" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "Error adding bucket tag: $error"
     return 1
@@ -481,30 +506,65 @@ put_bucket_tag() {
 # params:  bucket
 # export 'tags' on success, return 1 for error
 get_bucket_tags() {
-  if [ $# -ne 1 ]; then
-    echo "get bucket tag command missing bucket name"
+  if [ $# -ne 2 ]; then
+    echo "get bucket tag command missing command type, bucket name"
     return 1
   fi
   local result
-  tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$1") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
+    if [[ $tags =~ "No tags found" ]] || [[ $tags =~ "The TagSet does not exist" ]]; then
+      export tags=
+      return 0
+    fi
     echo "error getting bucket tags: $tags"
     return 1
   fi
   export tags
 }
 
+delete_bucket_tags() {
+  if [ $# -ne 2 ]; then
+    echo "delete bucket tag command missing command type, bucket name"
+    return 1
+  fi
+  local result
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api delete-bucket-tagging --bucket "$2" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag remove "$MC_ALIAS"/"$2" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
+  return 0
+}
+
 # add tags to object
 # params:  object, key, value
 # return:  0 for success, 1 for error
 put_object_tag() {
-  if [ $# -ne 4 ]; then
-    echo "object tag command missing object name, file, key, and/or value"
+  if [ $# -ne 5 ]; then
+    echo "object tag command missing command type, object name, file, key, and/or value"
     return 1
   fi
   local error
   local result
-  error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$1" --key "$2" --tagging "TagSet=[{Key=$3,Value=$4}]") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    error=$(aws --no-verify-ssl s3api put-object-tagging --bucket "$2" --key "$3" --tagging "TagSet=[{Key=$4,Value=$5}]" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    error=$(mc --insecure tag set "$MC_ALIAS"/"$2"/"$3" "$4=$5" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "Error adding object tag: $error"
     return 1
@@ -516,12 +576,19 @@ put_object_tag() {
 # params:  bucket
 # export 'tags' on success, return 1 for error
 get_object_tags() {
-  if [ $# -ne 2 ]; then
-    echo "get object tag command missing bucket and/or key"
+  if [ $# -ne 3 ]; then
+    echo "get object tag command missing command type, bucket, and/or key"
     return 1
   fi
   local result
-  tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$1" --key "$2") || result=$?
+  if [[ $1 == 'aws' ]]; then
+    tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
+  elif [[ $1 == 'mc' ]]; then
+    tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?
+  else
+    echo "invalid command type $1"
+    return 1
+  fi
   if [[ $result -ne 0 ]]; then
     echo "error getting object tags: $tags"
     return 1

tests/util_mc.sh

@@ -10,12 +10,12 @@ delete_bucket_recursive_mc() {
   fi
   local exit_code=0
   local error
-  error=$(mc --insecure rm --recursive --force versity/"$1" 2>&1) || exit_code="$?"
+  error=$(mc --insecure rm --recursive --force "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?"
   if [[ $exit_code -ne 0 ]]; then
     echo "error deleting bucket contents: $error"
     return 1
   fi
-  error=$(mc --insecure rb versity/"$1" 2>&1) || exit_code="$?"
+  error=$(mc --insecure rb "$MC_ALIAS"/"$1" 2>&1) || exit_code="$?"
   if [[ $exit_code -ne 0 ]]; then
     echo "error deleting bucket: $error"
     return 1

tests/versity.sh

@@ -7,30 +7,34 @@ check_exe_params() {
   elif [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
     echo "No AWS secret access key set"
     return 1
-  elif [ -z "$VERSITY_EXE" ]; then
-    echo "No versity executable location set"
-    return 1
-  elif [ -z "$BACKEND" ]; then
-    echo "No backend parameter set (options: 'posix')"
-    return 1
   elif [ -z "$AWS_PROFILE" ]; then
     echo "No AWS profile set"
     return 1
-  elif [ -z "$LOCAL_FOLDER" ]; then
-    echo "No local storage folder set"
-    return 1
   elif [ -z "$AWS_ENDPOINT_URL" ]; then
     echo "No AWS endpoint URL set"
     return 1
+  elif [ -z "$MC_ALIAS" ]; then
+    echo "No mc alias set"
+    return 1
+  elif [[ $RUN_VERSITYGW != "true" ]] && [[ $RUN_VERSITYGW != "false" ]]; then
+    echo "RUN_VERSITYGW must be 'true' or 'false'"
+    return 1
+  fi
+  if [[ $RUN_VERSITYGW == "true" ]]; then
+    if [ -z "$LOCAL_FOLDER" ]; then
+      echo "No local storage folder set"
+      return 1
+    elif [ -z "$VERSITY_EXE" ]; then
+      echo "No versity executable location set"
+      return 1
+    elif [ -z "$BACKEND" ]; then
+      echo "No backend parameter set (options: 'posix')"
+      return 1
+    fi
   fi
 }
 
 start_versity() {
-  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r tests/.secrets ]; then
-    source tests/.secrets
-  else
-    echo "Warning: no secrets file found"
-  fi
   if [ -z "$VERSITYGW_TEST_ENV" ]; then
     if [ -r tests/.env ]; then
       source tests/.env
@@ -41,6 +45,12 @@ start_versity() {
     # shellcheck source=./.env.default
     source "$VERSITYGW_TEST_ENV"
   fi
+  if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
+    # shellcheck source=/.secrets
+    source "$SECRETS_FILE"
+  else
+    echo "Warning: no secrets file found"
+  fi
 
   check_exe_params || check_result=$?
   if [[ $check_result -ne 0 ]]; then
@@ -48,25 +58,25 @@ start_versity() {
     return 1
   fi
 
-  base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY $VERSITY_EXE"
-  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
-    base_command+=" --cert $CERT --key $KEY"
-  fi
-  base_command+=" $BACKEND $LOCAL_FOLDER &"
-  eval "$base_command"
+  if [ "$RUN_VERSITYGW" == "true" ]; then
+    base_command="ROOT_ACCESS_KEY=$AWS_ACCESS_KEY_ID ROOT_SECRET_KEY=$AWS_SECRET_ACCESS_KEY VGW_REGION=$AWS_REGION $VERSITY_EXE"
+    if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+      base_command+=" --cert $CERT --key $KEY"
+    fi
+    base_command+=" $BACKEND $LOCAL_FOLDER &"
+    eval "$base_command"
+    versitygw_pid=$!
 
-  versitygw_pid=$!
-  export versitygw_pid \
-    AWS_ACCESS_KEY_ID \
-    AWS_SECRET_ACCESS_KEY \
-    VERSITY_EXE \
-    BACKEND \
-    AWS_PROFILE \
-    LOCAL_FOLDER \
-    AWS_ENDPOINT_URL
+    export versitygw_pid VERSITY_EXE BACKEND LOCAL_FOLDER
+  fi
+
+  export versitygw_pid AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
 }
 
 stop_versity() {
+  if [ "$RUN_VERSITYGW" == "false" ]; then
+    return
+  fi
   if [ -n "$versitygw_pid" ]; then
     if ps -p "$versitygw_pid" > /dev/null; then
       kill "$versitygw_pid"

versitygw.spec.in

@@ -1,31 +0,0 @@
-%global debug_package %{nil}
-%define pkg_version @@VERSION@@
-
-Name:		versitygw
-Version:	%{pkg_version}
-Release:	1%{?dist}
-Summary:	Versity S3 Gateway
-
-License:	Apache-2.0
-URL:		https://github.com/versity/versitygw
-Source0:	%{name}-%{version}.tar.gz
-%description
-The S3 gateway is an S3 protocol translator that allows an S3 client
-to access the supported backend storage as if it was a native S3 service.
-
-BuildRequires:	golang >= 1.20
-
-%prep
-%setup
-
-%build
-make
-
-%install
-mkdir -p %{buildroot}%{_bindir}
-install -m 0755 %{name} %{buildroot}%{_bindir}/
-
-%post
-
-%files
-%{_bindir}/%{name}