feat: Fixes all struct fields alignment issues, Adds betteralign check in the pipeline, to check the structs fields alignment

Test/GitHub actions speedup

.github/workflows/betteralign.yml

@@ -0,0 +1,23 @@
+name: betteralign
+on: pull_request
+jobs:
+  build:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
+        id: go
+
+      - name: Install betteralign
+        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
+
+      - name: Run betteralign
+        run: betteralign -test_files ./...

.github/workflows/system.yml

@@ -8,138 +8,96 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: "s3cmd, posix"
-            LOCAL_FOLDER: /tmp/gw1
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
+          - set: "mc, posix, non-file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam1
-            AWS_ENDPOINT_URL: https://127.0.0.1:7070
-            RUN_SET: "s3cmd"
+            RUN_SET: "mc-non-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7070
             BACKEND: "posix"
-          - set: "s3, posix"
-            LOCAL_FOLDER: /tmp/gw2
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
+          - set: "mc, posix, file count, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam2
-            AWS_ENDPOINT_URL: https://127.0.0.1:7071
-            RUN_SET: "s3"
+            RUN_SET: "mc-file-count"
             RECREATE_BUCKETS: "true"
-            PORT: 7071
             BACKEND: "posix"
-          - set: "s3api non-policy, posix"
-            LOCAL_FOLDER: /tmp/gw3
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
+          - set: "REST, posix, non-static, all, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam3
-            AWS_ENDPOINT_URL: https://127.0.0.1:7072
-            RUN_SET: "s3api"
-            RECREATE_BUCKETS: "true"
-            PORT: 7072
-            BACKEND: "posix"
-          - set: "mc, posix"
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam4
-            AWS_ENDPOINT_URL: https://127.0.0.1:7073
-            RUN_SET: "mc"
-            RECREATE_BUCKETS: "true"
-            PORT: 7073
-            BACKEND: "posix"
-          - set: "s3api-user, posix, s3 IAM"
-            LOCAL_FOLDER: /tmp/gw5
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
-            IAM_TYPE: s3
-            USERS_BUCKET: versity-gwtest-iam
-            AWS_ENDPOINT_URL: https://127.0.0.1:7074
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            PORT: 7074
-            BACKEND: "posix"
-          - set: "s3api non-policy, static buckets"
-            LOCAL_FOLDER: /tmp/gw6
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam6
-            AWS_ENDPOINT_URL: https://127.0.0.1:7075
-            RUN_SET: "s3api-non-policy"
-            RECREATE_BUCKETS: "false"
-            PORT: 7075
-            BACKEND: "posix"
-          - set: "s3api non-policy, s3 backend"
-            LOCAL_FOLDER: /tmp/gw7
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam7
-            AWS_ENDPOINT_URL: https://127.0.0.1:7076
-            RUN_SET: "s3api"
-            RECREATE_BUCKETS: "true"
-            PORT: 7076
-            BACKEND: "s3"
-          - set: "REST, posix"
-            LOCAL_FOLDER: /tmp/gw8
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam8
-            AWS_ENDPOINT_URL: https://127.0.0.1:7077
             RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
-            PORT: 7077
             BACKEND: "posix"
-          - set: "s3api policy, static buckets"
-            LOCAL_FOLDER: /tmp/gw9
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
+          - set: "s3, posix, non-file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-non-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, user, non-static, s3 IAM"
+            IAM_TYPE: s3
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3api, posix, bucket, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-bucket"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, multipart, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-multipart"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, object, static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-object"
+            RECREATE_BUCKETS: "false"
+            BACKEND: "posix"
+          - set: "s3api, posix, policy, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam9
-            AWS_ENDPOINT_URL: https://127.0.0.1:7078
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
-            PORT: 7078
             BACKEND: "posix"
-          - set: "s3api user, static buckets"
-            LOCAL_FOLDER: /tmp/gw10
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
+          - set: "s3api, posix, user, static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam10
-            AWS_ENDPOINT_URL: https://127.0.0.1:7079
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
-            PORT: 7079
             BACKEND: "posix"
-          - set: "s3api policy and user, posix"
-            LOCAL_FOLDER: /tmp/gw11
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-10
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-10
+          - set: "s3api, s3, multipart|object, non-static, folder IAM"
             IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam11
-            AWS_ENDPOINT_URL: https://127.0.0.1:7080
-            RUN_SET: "s3api-policy,s3api-user"
+            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
             RECREATE_BUCKETS: "true"
-            PORT: 7080
-            BACKEND: "posix"
-          - set: "s3api policy and user, s3 backend"
-            LOCAL_FOLDER: /tmp/gw12
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-11
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-11
-            IAM_TYPE: folder
-            USERS_FOLDER: /tmp/iam12
-            AWS_ENDPOINT_URL: https://127.0.0.1:7081
-            RUN_SET: "s3api-policy,s3api-user"
-            RECREATE_BUCKETS: "true"
-            PORT: 7081
             BACKEND: "s3"
+          - set: "s3api, s3, policy|user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "s3"
+          - set: "s3cmd, posix, file count, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-file-count"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, non-user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-non-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
+          - set: "s3cmd, posix, user, non-static, folder IAM"
+            IAM_TYPE: folder
+            RUN_SET: "s3cmd-user"
+            RECREATE_BUCKETS: "true"
+            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -176,15 +134,8 @@ jobs:
 
       - name: Build and run
         env:
-          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
-          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
-          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
-          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
-          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
-          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
-          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -192,6 +143,13 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
+          LOCAL_FOLDER: /tmp/gw
+          BUCKET_ONE_NAME: versity-gwtest-bucket-one
+          BUCKET_TWO_NAME: versity-gwtest-bucket-two
+          USERS_FOLDER: /tmp/iam
+          USERS_BUCKET: versity-gwtest-iam
+          AWS_ENDPOINT_URL: https://127.0.0.1:7070
+          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
@@ -204,6 +162,7 @@ jobs:
           REMOVE_TEST_FILE_FOLDER: true
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
+          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -224,6 +183,9 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
+      - name: Time report
+        run: cat ${{ github.workspace }}/time.log
+
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

auth/acl.go

@@ -45,18 +45,18 @@ type GetBucketAclOutput struct {
 
 type PutBucketAclInput struct {
 	Bucket              *string
-	ACL                 types.BucketCannedACL
 	AccessControlPolicy *AccessControlPolicy
 	GrantFullControl    *string
 	GrantRead           *string
 	GrantReadACP        *string
 	GrantWrite          *string
 	GrantWriteACP       *string
+	ACL                 types.BucketCannedACL
 }
 
 type AccessControlPolicy struct {
-	AccessControlList AccessControlList `xml:"AccessControlList"`
 	Owner             *types.Owner
+	AccessControlList AccessControlList `xml:"AccessControlList"`
 }
 
 type AccessControlList struct {
@@ -352,13 +352,13 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 }
 
 type AccessOptions struct {
-	Acl           ACL
 	AclPermission types.Permission
-	IsRoot        bool
-	Acc           Account
 	Bucket        string
 	Object        string
 	Action        Action
+	Acl           ACL
+	Acc           Account
+	IsRoot        bool
 	Readonly      bool
 }
 

auth/bucket_policy.go

@@ -63,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
 }
 
 type BucketPolicyItem struct {
-	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
+	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {

auth/iam.go

@@ -93,7 +93,6 @@ var (
 )
 
 type Opts struct {
-	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -119,11 +118,12 @@ type Opts struct {
 	S3Region               string
 	S3Bucket               string
 	S3Endpoint             string
+	RootAccount            Account
+	CacheTTL               int
+	CachePrune             int
 	S3DisableSSlVerfiy     bool
 	S3Debug                bool
 	CacheDisable           bool
-	CacheTTL               int
-	CachePrune             int
 }
 
 func New(o *Opts) (IAMService, error) {

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	value Account
 	exp   time.Time
+	value Account
 }
 
 type icache struct {
-	sync.RWMutex
-	expire time.Duration
 	items  map[string]item
+	expire time.Duration
+	sync.RWMutex
 }
 
 func (i *icache) set(k string, v Account) {

auth/iam_internal.go

@@ -33,6 +33,8 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
+	dir     string
+	rootAcc Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -40,8 +42,6 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
-	dir     string
-	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored

auth/iam_s3_object.go

@@ -42,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	client *s3.Client
+
+	access   string
+	secret   string
+	region   string
+	bucket   string
+	endpoint string
+	rootAcc  Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -50,15 +58,8 @@ type IAMServiceS3 struct {
 	// gateway instance if possible.
 	sync.RWMutex
 
-	access        string
-	secret        string
-	region        string
-	bucket        string
-	endpoint      string
 	sslSkipVerify bool
 	debug         bool
-	rootAcc       Account
-	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}

auth/object_lock.go

@@ -29,9 +29,9 @@ import (
 )
 
 type BucketLockConfig struct {
-	Enabled          bool
 	DefaultRetention *types.DefaultRetention
 	CreatedAt        *time.Time
+	Enabled          bool
 }
 
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {

aws/signer/v4/v4.go

@@ -85,6 +85,10 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
+
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -100,9 +104,6 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -117,8 +118,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	options      SignerOptions
 	keyDerivator keyDerivator
+	options      SignerOptions
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -133,17 +134,19 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
+	KeyDerivator keyDerivator
 	Request      *http.Request
+	Credentials  aws.Credentials
+	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
-	Time         v4Internal.SigningTime
-	Credentials  aws.Credentials
-	KeyDerivator keyDerivator
-	IsPreSign    bool
-	SignedHdrs   []string
 
 	PayloadHash string
 
+	SignedHdrs []string
+
+	IsPreSign bool
+
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/posix/posix.go

@@ -52,25 +52,25 @@ type Posix struct {
 	rootfd  *os.File
 	rootdir string
 
-	// chownuid/gid enable chowning of files to the account uid/gid
-	// when objects are uploaded
-	chownuid bool
-	chowngid bool
+	// bucket versioning directory path
+	versioningDir string
 
 	// euid/egid are the effective uid/gid of the running versitygw process
 	// used to determine if chowning is needed
 	euid int
 	egid int
 
+	// newDirPerm is the permission to set on newly created directories
+	newDirPerm fs.FileMode
+
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
+
 	// bucketlinks is a flag to enable symlinks to directories at the top
 	// level gateway directory to be treated as buckets the same as directories
 	bucketlinks bool
-
-	// bucket versioning directory path
-	versioningDir string
-
-	// newDirPerm is the permission to set on newly created directories
-	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &Posix{}
@@ -102,11 +102,11 @@ const (
 )
 
 type PosixOpts struct {
+	VersioningDir string
+	NewDirPerm    fs.FileMode
 	ChownUID      bool
 	ChownGID      bool
 	BucketLinks   bool
-	VersioningDir string
-	NewDirPerm    fs.FileMode
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {

backend/posix/with_otmpfile.go

@@ -38,12 +38,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
-	isOTmp     bool
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
+	isOTmp     bool
+	needsChown bool
 }
 
 var (

backend/scoutfs/scoutfs.go

@@ -48,12 +48,21 @@ type ScoutfsOpts struct {
 }
 
 type ScoutFS struct {
+
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
+
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
-	// bucket/object metadata storage facility
-	meta meta.MetadataStorer
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
 
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
@@ -70,14 +79,6 @@ type ScoutFS struct {
 	// when objects are uploaded
 	chownuid bool
 	chowngid bool
-
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
-
-	// newDirPerm is the permissions to use when creating new directories
-	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &ScoutFS{}

backend/scoutfs/scoutfs_compat.go

@@ -70,10 +70,10 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
-	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
+	needsChown bool
 }
 
 var (

backend/walk.go

@@ -27,10 +27,10 @@ import (
 )
 
 type WalkResults struct {
+	NextMarker     string
 	CommonPrefixes []types.CommonPrefix
 	Objects        []s3response.Object
 	Truncated      bool
-	NextMarker     string
 }
 
 type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
@@ -268,18 +268,18 @@ func contains(a string, strs []string) bool {
 }
 
 type WalkVersioningResults struct {
+	NextMarker          string
+	NextVersionIdMarker string
 	CommonPrefixes      []types.CommonPrefix
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	Truncated           bool
-	NextMarker          string
-	NextVersionIdMarker string
 }
 
 type ObjVersionFuncResult struct {
+	NextVersionIdMarker string
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
-	NextVersionIdMarker string
 	Truncated           bool
 }
 

backend/walk_test.go

@@ -41,8 +41,8 @@ type testcase struct {
 	prefix    string
 	delimiter string
 	marker    string
-	maxObjs   int32
 	expected  backend.WalkResults
+	maxObjs   int32
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {

metrics/metrics.go

@@ -43,13 +43,14 @@ type Tag struct {
 
 // Manager is a manager of metrics plugins
 type Manager struct {
-	wg  sync.WaitGroup
 	ctx context.Context
 
+	addDataChan chan datapoint
+
 	config Config
 
-	publishers  []publisher
-	addDataChan chan datapoint
+	publishers []publisher
+	wg         sync.WaitGroup
 }
 
 type Config struct {
@@ -220,6 +221,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
 
 type datapoint struct {
 	key   string
-	value int64
 	tags  []Tag
+	value int64
 }

s3api/admin-server.go

@@ -26,11 +26,11 @@ import (
 )
 
 type S3AdminServer struct {
-	app     *fiber.App
 	backend backend.Backend
+	app     *fiber.App
 	router  *S3AdminRouter
-	port    string
 	cert    *tls.Certificate
+	port    string
 }
 
 func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {

s3api/controllers/admin_test.go

@@ -64,11 +64,11 @@ func TestAdminController_CreateUser(t *testing.T) {
 	`
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-create-user-malformed-body",
@@ -149,11 +149,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 	`
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-update-user-success",
@@ -223,11 +223,11 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app.Patch("/delete-user", adminController.DeleteUser)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -280,11 +280,11 @@ func TestAdminController_ListUsers(t *testing.T) {
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Admin-list-users-iam-error",
@@ -361,11 +361,11 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -424,11 +424,11 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "List-buckets-success",

s3api/controllers/base.go

@@ -3308,14 +3308,14 @@ type MetaOpts struct {
 	Logger        s3log.AuditLogger
 	EvSender      s3event.S3EventSender
 	MetricsMng    *metrics.Manager
-	ContentLength int64
-	Action        string
-	BucketOwner   string
-	ObjectSize    int64
-	ObjectCount   int64
-	EventName     s3event.EventType
 	ObjectETag    *string
 	VersionId     *string
+	Action        string
+	BucketOwner   string
+	EventName     s3event.EventType
+	ContentLength int64
+	ObjectSize    int64
+	ObjectCount   int64
 	Status        int
 }
 

s3api/controllers/base_test.go

@@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
-		name       string
 		args       args
 		app        *fiber.App
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -233,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -435,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -728,11 +728,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -1034,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -1243,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1334,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1432,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1541,11 +1541,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1643,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Head-object-success",
@@ -1723,11 +1723,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
 	tests := []struct {
-		name       string
 		app        *fiber.App
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Restore-object-success",
@@ -1808,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",
@@ -1883,10 +1883,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		name       string
 		args       args
-		wantErr    bool
+		name       string
 		statusCode int
+		wantErr    bool
 	}{
 		{
 			name: "Internal-server-error",

s3api/middlewares/authentication.go

@@ -149,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	root RootUserConfig
 	iam  auth.IAMService
+	root RootUserConfig
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		name string
-		sa   *S3ApiRouter
 		args args
+		sa   *S3ApiRouter
+		name string
 	}{
 		{
 			name: "Initialize S3 api router",

s3api/server.go

@@ -29,15 +29,15 @@ import (
 )
 
 type S3ApiServer struct {
-	app      *fiber.App
 	backend  backend.Backend
+	app      *fiber.App
 	router   *S3ApiRouter
-	port     string
 	cert     *tls.Certificate
+	port     string
+	health   string
 	quiet    bool
 	debug    bool
 	readonly bool
-	health   string
 }
 
 func New(

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		name            string
-		args            args
 		wantS3ApiServer *S3ApiServer
+		args            args
+		name            string
 		wantErr         bool
 	}{
 		{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		name    string
 		sa      *S3ApiServer
+		name    string
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
+	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
-	r      *HashReader
 	debug  bool
 }
 

s3api/utils/chunk-reader.go

@@ -48,15 +48,15 @@ const (
 // object data stream
 type ChunkReader struct {
 	r                io.Reader
-	signingKey       []byte
+	chunkHash        hash.Hash
 	prevSig          string
 	parsedSig        string
+	strToSignPrefix  string
+	signingKey       []byte
+	stash            []byte
 	currentChunkSize int64
 	chunkDataLeft    int64
 	trailerExpected  int
-	stash            []byte
-	chunkHash        hash.Hash
-	strToSignPrefix  string
 	skipcheck        bool
 }
 

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
+	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
-	r      io.Reader
 	debug  bool
 }
 

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		str  string
 		date time.Time
+		str  string
 	}
 	tests := []struct {
-		name string
 		args args
 		err  error
+		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/utils_test.go

@@ -49,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
 
 	tests := []struct {
-		name    string
 		args    args
 		want    *http.Request
-		wantErr bool
+		name    string
 		hdrs    []string
+		wantErr bool
 	}{
 		{
 			name: "Success-response",
@@ -101,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) {
 	req := ctx.Request()
 
 	tests := []struct {
-		name         string
 		args         args
 		wantMetadata map[string]string
+		name         string
 	}{
 		{
 			name: "Success-empty-response",

s3event/event.go

@@ -30,11 +30,11 @@ type S3EventSender interface {
 }
 
 type EventMeta struct {
+	ObjectETag  *string
+	VersionId   *string
 	BucketOwner string
 	EventName   EventType
 	ObjectSize  int64
-	ObjectETag  *string
-	VersionId   *string
 }
 
 type EventSchema struct {
@@ -42,6 +42,8 @@ type EventSchema struct {
 }
 
 type EventRecord struct {
+	ResponseElements  EventResponseElements `json:"responseElements"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
 	AwsRegion         string                `json:"awsRegion"`
@@ -49,9 +51,7 @@ type EventRecord struct {
 	EventName         EventType             `json:"eventName"`
 	UserIdentity      EventUserIdentity     `json:"userIdentity"`
 	RequestParameters EventRequestParams    `json:"requestParameters"`
-	ResponseElements  EventResponseElements `json:"responseElements"`
 	S3                EventS3Data           `json:"s3"`
-	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 }
 
 type EventUserIdentity struct {
@@ -99,11 +99,11 @@ type EventS3BucketData struct {
 }
 
 type EventObjectData struct {
-	Key       string  `json:"key"`
-	Size      int64   `json:"size"`
 	ETag      *string `json:"eTag"`
 	VersionId *string `json:"versionId"`
+	Key       string  `json:"key"`
 	Sequencer string  `json:"sequencer"`
+	Size      int64   `json:"size"`
 }
 
 type EventConfig struct {

s3event/kafka.go

@@ -31,9 +31,9 @@ import (
 var sequencer = 0
 
 type Kafka struct {
-	key    string
 	writer *kafka.Writer
 	filter EventFilter
+	key    string
 	mu     sync.Mutex
 }
 

s3event/nats.go

@@ -27,10 +27,10 @@ import (
 )
 
 type NatsEventSender struct {
-	topic  string
 	client *nats.Conn
-	mu     sync.Mutex
 	filter EventFilter
+	topic  string
+	mu     sync.Mutex
 }
 
 func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {

s3event/webhook.go

@@ -30,9 +30,9 @@ import (
 )
 
 type Webhook struct {
-	url    string
 	client *http.Client
 	filter EventFilter
+	url    string
 	mu     sync.Mutex
 }
 

s3log/audit-logger.go

@@ -33,8 +33,8 @@ type AuditLogger interface {
 
 type LogMeta struct {
 	BucketOwner string
-	ObjectSize  int64
 	Action      string
+	ObjectSize  int64
 	HttpStatus  int
 }
 
@@ -45,21 +45,16 @@ type LogConfig struct {
 }
 
 type LogFields struct {
+	Time               time.Time
 	BucketOwner        string
 	Bucket             string
-	Time               time.Time
 	RemoteIP           string
 	Requester          string
 	RequestID          string
 	Operation          string
 	Key                string
 	RequestURI         string
-	HttpStatus         int
 	ErrorCode          string
-	BytesSent          int
-	ObjectSize         int64
-	TotalTime          int64
-	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	VersionID          string
@@ -71,6 +66,11 @@ type LogFields struct {
 	TLSVersion         string
 	AccessPointARN     string
 	AclRequired        string
+	HttpStatus         int
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
 }
 
 type AdminLogFields struct {
@@ -80,17 +80,17 @@ type AdminLogFields struct {
 	RequestID          string
 	Operation          string
 	RequestURI         string
-	HttpStatus         int
 	ErrorCode          string
-	BytesSent          int
-	TotalTime          int64
-	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	SignatureVersion   string
 	CipherSuite        string
 	AuthenticationType string
 	TLSVersion         string
+	HttpStatus         int
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
 }
 
 type Loggers struct {

s3log/file.go

@@ -34,10 +34,10 @@ const (
 
 // FileLogger is a local file audit log
 type FileLogger struct {
-	logfile string
 	f       *os.File
-	gotErr  bool
+	logfile string
 	mu      sync.Mutex
+	gotErr  bool
 }
 
 var _ AuditLogger = &FileLogger{}

s3log/webhook.go

@@ -33,8 +33,8 @@ import (
 
 // WebhookLogger is a webhook URL audit log
 type WebhookLogger struct {
-	mu  sync.Mutex
 	url string
+	mu  sync.Mutex
 }
 
 var _ AuditLogger = &WebhookLogger{}

s3response/s3response.go

@@ -35,17 +35,17 @@ type PutObjectOutput struct {
 
 // Part describes part metadata.
 type Part struct {
-	PartNumber   int
 	LastModified time.Time
 	ETag         string
+	PartNumber   int
 	Size         int64
 }
 
 func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Part
 	aux := &struct {
-		LastModified string `xml:"LastModified"`
 		*Alias
+		LastModified string `xml:"LastModified"`
 	}{
 		Alias: (*Alias)(&p),
 	}
@@ -59,23 +59,23 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
+	Initiator Initiator
+	Owner     Owner
+
 	Bucket   string
 	Key      string
 	UploadID string `xml:"UploadId"`
 
-	Initiator Initiator
-	Owner     Owner
-
 	// The class of storage used to store the object.
 	StorageClass types.StorageClass
 
+	// List of parts.
+	Parts []Part `xml:"Part"`
+
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
-
-	// List of parts.
-	Parts []Part `xml:"Part"`
 }
 
 type ObjectAttributes string
@@ -97,23 +97,23 @@ func (o ObjectAttributes) IsValid() bool {
 }
 
 type GetObjectAttributesResponse struct {
-	ETag         *string
-	ObjectSize   *int64
-	StorageClass types.StorageClass `xml:",omitempty"`
-	ObjectParts  *ObjectParts
+	ETag        *string
+	ObjectSize  *int64
+	ObjectParts *ObjectParts
 
 	// Not included in the response body
 	VersionId    *string
 	LastModified *time.Time
 	DeleteMarker *bool
+	StorageClass types.StorageClass `xml:",omitempty"`
 }
 
 type ObjectParts struct {
+	Parts                []types.ObjectPart `xml:"Part"`
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
-	Parts                []types.ObjectPart `xml:"Part"`
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
@@ -128,18 +128,17 @@ type ListMultipartUploadsResult struct {
 	Delimiter          string
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
-	MaxUploads         int
-	IsTruncated        bool
 
 	// List of pending uploads.
 	Uploads []Upload `xml:"Upload"`
 
 	// Delimed common prefixes.
 	CommonPrefixes []CommonPrefix
+	MaxUploads     int
+	IsTruncated    bool
 }
 
 type ListObjectsResult struct {
-	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name           *string
 	Prefix         *string
 	Marker         *string
@@ -147,13 +146,13 @@ type ListObjectsResult struct {
 	MaxKeys        *int32
 	Delimiter      *string
 	IsTruncated    *bool
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	EncodingType   types.EncodingType
 	Contents       []Object
 	CommonPrefixes []types.CommonPrefix
-	EncodingType   types.EncodingType
 }
 
 type ListObjectsV2Result struct {
-	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name                  *string
 	Prefix                *string
 	StartAfter            *string
@@ -163,9 +162,10 @@ type ListObjectsV2Result struct {
 	MaxKeys               *int32
 	Delimiter             *string
 	IsTruncated           *bool
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+	EncodingType          types.EncodingType
 	Contents              []Object
 	CommonPrefixes        []types.CommonPrefix
-	EncodingType          types.EncodingType
 }
 
 type Object struct {
@@ -197,19 +197,19 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // Upload describes in progress multipart upload
 type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
+	Initiated    time.Time
 	Initiator    Initiator
 	Owner        Owner
+	Key          string
+	UploadID     string `xml:"UploadId"`
 	StorageClass types.StorageClass
-	Initiated    time.Time
 }
 
 func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Upload
 	aux := &struct {
-		Initiated string `xml:"Initiated"`
 		*Alias
+		Initiated string `xml:"Initiated"`
 	}{
 		Alias: (*Alias)(&u),
 	}
@@ -261,11 +261,11 @@ type DeleteResult struct {
 }
 type SelectObjectContentPayload struct {
 	Expression          *string
-	ExpressionType      types.ExpressionType
 	RequestProgress     *types.RequestProgress
 	InputSerialization  *types.InputSerialization
 	OutputSerialization *types.OutputSerialization
 	ScanRange           *types.ScanRange
+	ExpressionType      types.ExpressionType
 }
 
 type SelectObjectContentResult struct {
@@ -283,30 +283,30 @@ type Bucket struct {
 
 type ListBucketsInput struct {
 	Owner             string
-	IsAdmin           bool
 	ContinuationToken string
 	Prefix            string
 	MaxBuckets        int32
+	IsAdmin           bool
 }
 
 type ListAllMyBucketsResult struct {
 	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
 	Owner             CanonicalUser
-	Buckets           ListAllMyBucketsList
 	ContinuationToken string `xml:"ContinuationToken,omitempty"`
 	Prefix            string `xml:"Prefix,omitempty"`
+	Buckets           ListAllMyBucketsList
 }
 
 type ListAllMyBucketsEntry struct {
-	Name         string
 	CreationDate time.Time
+	Name         string
 }
 
 func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias ListAllMyBucketsEntry
 	aux := &struct {
-		CreationDate string `xml:"CreationDate"`
 		*Alias
+		CreationDate string `xml:"CreationDate"`
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -335,8 +335,8 @@ type CopyObjectResult struct {
 func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias CopyObjectResult
 	aux := &struct {
-		LastModified string `xml:"LastModified"`
 		*Alias
+		LastModified string `xml:"LastModified"`
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -404,15 +404,15 @@ type ListVersionsResult struct {
 }
 
 type GetBucketVersioningOutput struct {
-	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 	MFADelete *types.MFADeleteStatus
 	Status    *types.BucketVersioningStatus
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 }
 
 type PutObjectRetentionInput struct {
+	RetainUntilDate AmzDate
 	XMLName         xml.Name `xml:"Retention"`
 	Mode            types.ObjectLockRetentionMode
-	RetainUntilDate AmzDate
 }
 
 type AmzDate struct {

s3select/message-handler.go

@@ -204,7 +204,6 @@ func genErrorMessage(errorCode, errorMessage string) []byte {
 type GetProgress func() (bytesScanned int64, bytesProcessed int64)
 
 type MessageHandler struct {
-	sync.Mutex
 	ctx           context.Context
 	cancel        context.CancelFunc
 	writer        *bufio.Writer
@@ -213,6 +212,7 @@ type MessageHandler struct {
 	stopCh        chan bool
 	resetCh       chan bool
 	bytesReturned int64
+	sync.Mutex
 }
 
 // NewMessageHandler creates a new MessageHandler instance and starts the event streaming

tests/.env.default

@@ -28,4 +28,5 @@ PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
 VERSIONING_DIR=/tmp/versioning
-COMMAND_LOG=command.log
+COMMAND_LOG=command.log
+TIME_LOG=time.log

tests/docker-compose-bats.yml

@@ -27,7 +27,7 @@ services:
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
-    command: ["s3api-non-policy"]
+    command: ["s3api-bucket,s3api-object"]
   direct:
     build:
       dockerfile: tests/Dockerfile_direct

tests/integration/bench.go

@@ -26,9 +26,9 @@ import (
 )
 
 type prefResult struct {
+	err     error
 	elapsed time.Duration
 	size    int64
-	err     error
 }
 
 func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {

tests/integration/data-io.go

@@ -22,9 +22,9 @@ import (
 )
 
 type RReader struct {
+	hash     hash.Hash
 	buf      []byte
 	dataleft int
-	hash     hash.Hash
 }
 
 func NewDataReader(totalsize, bufsize int) *RReader {

tests/integration/s3conf.go

@@ -35,10 +35,10 @@ type S3Conf struct {
 	awsSecret         string
 	awsRegion         string
 	endpoint          string
-	checksumDisable   bool
-	pathStyle         bool
 	PartSize          int64
 	Concurrency       int
+	checksumDisable   bool
+	pathStyle         bool
 	debug             bool
 	versioningEnabled bool
 	azureTests        bool

tests/integration/utils.go

@@ -168,9 +168,9 @@ func teardown(s *S3Conf, bucket string) error {
 }
 
 type setupCfg struct {
-	LockEnabled      bool
 	VersioningStatus types.BucketVersioningStatus
 	Ownership        types.ObjectOwnership
+	LockEnabled      bool
 }
 
 type setupOpt func(*setupCfg)
@@ -214,12 +214,12 @@ func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client,
 }
 
 type authConfig struct {
+	date     time.Time
 	testName string
 	path     string
 	method   string
-	body     []byte
 	service  string
-	date     time.Time
+	body     []byte
 }
 
 func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
@@ -434,9 +434,9 @@ func contains(s []string, e string) bool {
 }
 
 type putObjectOutput struct {
-	csum [32]byte
-	data []byte
 	res  *s3.PutObjectOutput
+	data []byte
+	csum [32]byte
 }
 
 func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {

tests/rest_scripts/get_bucket_tagging.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="GET
+/$bucket_name
+tagging=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?tagging="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_bucket_tagging.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+# shellcheck disable=SC2153
+key="$TAG_KEY"
+# shellcheck disable=SC2153
+value="$TAG_VALUE"
+
+payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+   <TagSet>
+      <Tag>
+         <Key>$key</Key>
+         <Value>$value</Value>
+      </Tag>
+   </TagSet>
+</Tagging>"
+
+content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
+payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="PUT
+/$bucket_name
+tagging=
+content-md5:$content_md5
+host:$host
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+content-md5;host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?tagging="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"Content-MD5: $content_md5\""
+-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\""
+-d "\"${payload//\"/\\\"}\""
+-o "$OUTPUT_FILE")
+
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/run.sh

@@ -19,14 +19,21 @@ show_help() {
     echo "Usage: $0 [option...]"
     echo "   -h, --help          Display this help message and exit"
     echo "                       Separate the below by comma"
-    echo "   s3api               Run tests with s3api cli"
-    echo "   s3api-non-policy    Run policy tests with s3api cli"
+    echo "   s3api               Run all tests with s3api cli"
+    echo "   s3api-multipart     Run multipart tests with s3api cli"
+    echo "   s3api-bucket        Run bucket tests with s3api cli"
+    echo "   s3api-object        Run object tests with s3api cli"
     echo "   s3api-policy        Run policy tests with s3api cli"
+    echo "   s3api-user          Run user tests with s3api cli"
     echo "   s3                  Run tests with s3 cli"
     echo "   s3cmd               Run tests with s3cmd utility"
+    echo "   s3cmd-user          Run user tests with s3cmd utility"
+    echo "   s3cmd-non-user      Run non-user tests with s3cmd utility"
+    echo "   s3cmd-file-count    Run file count test with s3cmd utility"
     echo "   mc                  Run tests with mc utility"
+    echo "   mc-non-file-count   Run non-file count tests with mc utility"
+    echo "   mc-file-count       Run file count test with mc utility"
     echo "   rest                Run tests with rest cli"
-    echo "   s3api-user          Run user tests with aws cli"
 }
 
 handle_param() {
@@ -35,7 +42,7 @@ handle_param() {
           show_help
           exit 0
           ;;
-      s3|s3api|s3cmd|mc|s3api-user|rest|s3api-policy|s3api-non-policy)
+      s3|s3-file-count|s3-non-file-count|s3api|s3cmd|s3cmd-user|s3cmd-non-user|s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|s3api-bucket|s3api-object|s3api-multipart)
           run_suite "$1"
           ;;
       *) # Handle unrecognized options or positional arguments
@@ -50,25 +57,50 @@ run_suite() {
   case $1 in
     s3api)
       echo "Running all s3api tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
+      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
+      fi
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       fi
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
+      fi
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
       fi
       ;;
+    s3api-multipart)
+      echo "Running s3api multipart tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
+      ;;
     s3api-policy)
       echo "Running s3api policy tests ..."
       "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       ;;
-    s3api-non-policy)
-      echo "Running s3api non-policy tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
+    s3api-bucket)
+      echo "Running s3api bucket tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
+      ;;
+    s3api-object)
+      echo "Running s3api object tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
       ;;
     s3)
       echo "Running s3 tests ..."
       "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
+      fi
+      ;;
+    s3-non-file-count)
+      echo "Running s3 non-file count tests ..."
+      "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
+      ;;
+    s3-file-count)
+      echo "Running s3 file count test ..."
+      "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
       ;;
     s3cmd)
       echo "Running s3cmd tests ..."
@@ -76,10 +108,36 @@ run_suite() {
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
       fi
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
+      fi
+      ;;
+    s3cmd-user)
+      echo "Running s3cmd user tests ..."
+      "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
+      ;;
+    s3cmd-non-user)
+      echo "Running s3cmd non-user tests ..."
+      "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
+      ;;
+    s3cmd-file-count)
+      echo "Running s3cmd file count test ..."
+      "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
       ;;
     mc)
       echo "Running mc tests ..."
       "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+      if [[ $exit_code -eq 0 ]]; then
+        "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
+      fi
+      ;;
+    mc-non-file-count)
+      echo "Running mc non-file count tests ..."
+      "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
+      ;;
+    mc-file-count)
+      echo "Running mc file count test ..."
+      "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
       ;;
     rest)
       echo "Running rest tests ..."

tests/setup.sh

@@ -28,7 +28,7 @@ setup() {
   base_setup
 
   log 4 "Running test $BATS_TEST_NAME"
-  if [[ $LOG_LEVEL -ge 5 ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
     start_time=$(date +%s)
     export start_time
   fi
@@ -75,9 +75,13 @@ teardown() {
     fi
   fi
   stop_versity
-  if [[ $LOG_LEVEL -ge 5 ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
     end_time=$(date +%s)
-    log 4 "Total test time: $((end_time - start_time))"
+    total_time=$((end_time - start_time))
+    log 4 "Total test time: $total_time"
+    if [[ -n "$TIME_LOG" ]]; then
+      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
+    fi
   fi
   if [[ -n "$COVERAGE_DB" ]]; then
     record_result

tests/test_mc.sh

@@ -112,10 +112,6 @@ export RUN_MC=true
   test_common_presigned_url_utf8_chars "mc"
 }
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "mc"
-}
-
 @test "test_create_bucket_invalid_name_mc" {
   if [[ $RECREATE_BUCKETS != "true" ]]; then
     return

tests/test_mc_file_count.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+
+export RUN_MC=true
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}

tests/test_rest.sh

@@ -373,3 +373,25 @@ source ./tests/util_versioning.sh
   run add_and_check_checksum "$TEST_FILE_FOLDER/$test_file" "$test_file"
   assert_success
 }
+
+@test "REST - bucket tagging - no tags" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run verify_no_bucket_tags_rest "$BUCKET_ONE_NAME"
+  assert_success
+}
+
+@test "REST - bucket tagging - tags" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/932"
+  fi
+  test_key="testKey"
+  test_value="testValue"
+
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run add_verify_bucket_tags_rest "$BUCKET_ONE_NAME" "$test_key" "$test_value"
+  assert_success
+}

tests/test_s3.sh

@@ -51,10 +51,6 @@ source ./tests/util_file.sh
   test_common_list_buckets "s3"
 }
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3"
-}
-
 @test "test_delete_bucket" {
   if [[ $RECREATE_BUCKETS == "false" ]]; then
     skip "will not test bucket deletion in static bucket test config"

tests/test_s3_file_count.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+source ./tests/util_file.sh
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3"
+}

tests/test_s3api_bucket.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/setup.sh
+source ./tests/util.sh
+source ./tests/util_aws.sh
+source ./tests/util_create_bucket.sh
+source ./tests/util_file.sh
+source ./tests/util_lock_config.sh
+source ./tests/util_tags.sh
+source ./tests/util_users.sh
+source ./tests/test_aws_root_inner.sh
+source ./tests/test_common.sh
+source ./tests/test_common_acl.sh
+source ./tests/commands/copy_object.sh
+source ./tests/commands/delete_bucket_policy.sh
+source ./tests/commands/delete_object_tagging.sh
+source ./tests/commands/get_bucket_acl.sh
+source ./tests/commands/get_bucket_policy.sh
+source ./tests/commands/get_bucket_versioning.sh
+source ./tests/commands/get_object.sh
+source ./tests/commands/get_object_attributes.sh
+source ./tests/commands/get_object_legal_hold.sh
+source ./tests/commands/get_object_lock_configuration.sh
+source ./tests/commands/get_object_retention.sh
+source ./tests/commands/get_object_tagging.sh
+source ./tests/commands/list_object_versions.sh
+source ./tests/commands/put_bucket_acl.sh
+source ./tests/commands/put_bucket_policy.sh
+source ./tests/commands/put_bucket_versioning.sh
+source ./tests/commands/put_object.sh
+source ./tests/commands/put_object_legal_hold.sh
+source ./tests/commands/put_object_lock_configuration.sh
+source ./tests/commands/put_object_retention.sh
+source ./tests/commands/put_public_access_block.sh
+source ./tests/commands/select_object_content.sh
+
+export RUN_USERS=true
+
+# create-bucket
+@test "test_create_delete_bucket_aws" {
+  test_common_create_delete_bucket "aws"
+}
+
+@test "test_create_bucket_invalid_name" {
+  test_create_bucket_invalid_name_aws_root
+}
+
+# delete-bucket - test_create_delete_bucket_aws
+
+# delete-bucket-policy
+@test "test_get_put_delete_bucket_policy" {
+  if [[ -n $SKIP_POLICY ]]; then
+    skip "will not test policy actions with SKIP_POLICY set"
+  fi
+  test_common_get_put_delete_bucket_policy "aws"
+}
+
+# delete-bucket-tagging
+@test "test-set-get-delete-bucket-tags" {
+  test_common_set_get_delete_bucket_tags "aws"
+}
+
+# get-bucket-acl
+@test "test_get_bucket_acl" {
+  test_get_bucket_acl_aws_root
+}
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "aws"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging - test_set_get_delete_bucket_tags
+
+@test "test_head_bucket_invalid_name" {
+  if head_bucket "aws" ""; then
+    fail "able to get bucket info for invalid name"
+  fi
+}
+
+# test listing buckets on versitygw
+@test "test_list_buckets" {
+  test_common_list_buckets "s3api"
+}
+
+@test "test_put_bucket_acl" {
+  test_common_put_bucket_acl "s3api"
+}
+
+@test "test_head_bucket" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
+  log 5 "INFO:  $bucket_info"
+  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
+  [[ $region != "" ]] || fail "empty bucket region"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_head_bucket_doesnt_exist" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}

tests/test_s3api_multipart.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/setup.sh
+source ./tests/test_aws_root_inner.sh
+source ./tests/util_file.sh
+source ./tests/util_multipart.sh
+source ./tests/util_tags.sh
+source ./tests/commands/get_object.sh
+source ./tests/commands/put_object.sh
+source ./tests/commands/list_multipart_uploads.sh
+
+# abort-multipart-upload
+@test "test_abort_multipart_upload" {
+  test_abort_multipart_upload_aws_root
+}
+
+# complete-multipart-upload
+@test "test_complete_multipart_upload" {
+  test_complete_multipart_upload_aws_root
+}
+
+# create-multipart-upload
+@test "test_create_multipart_upload_properties" {
+  test_create_multipart_upload_properties_aws_root
+}
+
+# test multi-part upload list parts command
+@test "test-multipart-upload-list-parts" {
+  test_multipart_upload_list_parts_aws_root
+}
+
+# test listing of active uploads
+@test "test-multipart-upload-list-uploads" {
+  local bucket_file_one="bucket-file-one"
+  local bucket_file_two="bucket-file-two"
+
+  if [[ $RECREATE_BUCKETS == false ]]; then
+    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  run create_test_files "$bucket_file_one" "$bucket_file_two"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
+  assert_success
+}
+
+@test "test-multipart-upload-from-bucket" {
+  local bucket_file="bucket-file"
+
+  run create_test_file "$bucket_file"
+  assert_success
+
+  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
+  assert_success
+
+  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
+  assert_success
+
+  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_too_large" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_valid" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
+  assert_success
+}

tests/test_s3api_object.sh

@@ -20,7 +20,6 @@ source ./tests/util_aws.sh
 source ./tests/util_create_bucket.sh
 source ./tests/util_file.sh
 source ./tests/util_lock_config.sh
-source ./tests/util_multipart.sh
 source ./tests/util_tags.sh
 source ./tests/util_users.sh
 source ./tests/test_aws_root_inner.sh
@@ -38,7 +37,6 @@ source ./tests/commands/get_object_legal_hold.sh
 source ./tests/commands/get_object_lock_configuration.sh
 source ./tests/commands/get_object_retention.sh
 source ./tests/commands/get_object_tagging.sh
-source ./tests/commands/list_multipart_uploads.sh
 source ./tests/commands/list_object_versions.sh
 source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/put_bucket_policy.sh
@@ -52,16 +50,6 @@ source ./tests/commands/select_object_content.sh
 
 export RUN_USERS=true
 
-# abort-multipart-upload
-@test "test_abort_multipart_upload" {
-  test_abort_multipart_upload_aws_root
-}
-
-# complete-multipart-upload
-@test "test_complete_multipart_upload" {
-  test_complete_multipart_upload_aws_root
-}
-
 # copy-object
 @test "test_copy_object" {
   test_common_copy_object "s3api"
@@ -71,35 +59,6 @@ export RUN_USERS=true
   copy_object_empty || fail "copy objects with no parameters test failure"
 }
 
-# create-bucket
-@test "test_create_delete_bucket_aws" {
-  test_common_create_delete_bucket "aws"
-}
-
-@test "test_create_bucket_invalid_name" {
-  test_create_bucket_invalid_name_aws_root
-}
-
-# create-multipart-upload
-@test "test_create_multipart_upload_properties" {
-  test_create_multipart_upload_properties_aws_root
-}
-
-# delete-bucket - test_create_delete_bucket_aws
-
-# delete-bucket-policy
-@test "test_get_put_delete_bucket_policy" {
-  if [[ -n $SKIP_POLICY ]]; then
-    skip "will not test policy actions with SKIP_POLICY set"
-  fi
-  test_common_get_put_delete_bucket_policy "aws"
-}
-
-# delete-bucket-tagging
-@test "test-set-get-delete-bucket-tags" {
-  test_common_set_get_delete_bucket_tags "aws"
-}
-
 # delete-object - tested with bucket cleanup before or after tests
 
 # delete-object-tagging
@@ -115,20 +74,6 @@ export RUN_USERS=true
   test_delete_objects_aws_root
 }
 
-# get-bucket-acl
-@test "test_get_bucket_acl" {
-  test_get_bucket_acl_aws_root
-}
-
-# get-bucket-location
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "aws"
-}
-
-# get-bucket-policy - test_get_put_delete_bucket_policy
-
-# get-bucket-tagging - test_set_get_delete_bucket_tags
-
 # get-object
 @test "test_get_object_full_range" {
   test_get_object_full_range_aws_root
@@ -143,12 +88,6 @@ export RUN_USERS=true
   test_get_object_attributes_aws_root
 }
 
-@test "test_head_bucket_invalid_name" {
-  if head_bucket "aws" ""; then
-    fail "able to get bucket info for invalid name"
-  fi
-}
-
 @test "test_put_object" {
   test_put_object_aws_root
 }
@@ -168,11 +107,6 @@ export RUN_USERS=true
   test_common_put_object_no_data "aws"
 }
 
-# test listing buckets on versitygw
-@test "test_list_buckets" {
-  test_common_list_buckets "s3api"
-}
-
 # test listing a bucket's objects on versitygw
 @test "test_list_objects" {
   test_common_list_objects "aws"
@@ -186,10 +120,6 @@ export RUN_USERS=true
   test_get_put_object_retention_aws_root
 }
 
-@test "test_put_bucket_acl" {
-  test_common_put_bucket_acl "s3api"
-}
-
 # test v1 s3api list objects command
 @test "test-s3api-list-objects-v1" {
   test_s3api_list_objects_v1_aws_root
@@ -205,77 +135,6 @@ export RUN_USERS=true
   test_common_set_get_object_tags "aws"
 }
 
-# test multi-part upload list parts command
-@test "test-multipart-upload-list-parts" {
-  test_multipart_upload_list_parts_aws_root
-}
-
-# test listing of active uploads
-@test "test-multipart-upload-list-uploads" {
-  local bucket_file_one="bucket-file-one"
-  local bucket_file_two="bucket-file-two"
-
-  if [[ $RECREATE_BUCKETS == false ]]; then
-    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
-    assert_success
-  fi
-
-  run create_test_files "$bucket_file_one" "$bucket_file_two"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
-  assert_success
-}
-
-@test "test-multipart-upload-from-bucket" {
-  local bucket_file="bucket-file"
-
-  run create_test_file "$bucket_file"
-  assert_success
-
-  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
-  assert_success
-
-  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
-  assert_success
-
-  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_too_large" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_valid" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
-  assert_success
-}
-
 @test "test-presigned-url-utf8-chars" {
   test_common_presigned_url_utf8_chars "aws"
 }
@@ -293,18 +152,11 @@ export RUN_USERS=true
   run setup_bucket "aws" "$BUCKET_ONE_NAME"
   assert_success
 
-  put_object "aws" "$TEST_FILE_FOLDER/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || fail "failed to add object to bucket"
+  run put_object "aws" "$TEST_FILE_FOLDER/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name"
+  assert_success
 
-  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"
-  prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1) || fail "error getting object prefix from object list: $prefix"
-  [[ $prefix == "$folder_name/" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
-
-  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"
-  key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key"
-  [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)"
-
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files $folder_name
+  run check_object_listing_with_prefixes "$BUCKET_ONE_NAME" "$folder_name" "$object_name"
+  assert_success
 }
 
 # ensure that lists of files greater than a size of 1000 (pagination) are returned properly
@@ -331,31 +183,10 @@ export RUN_USERS=true
 #  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
 #}
 
-@test "test_head_bucket" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
-  log 5 "INFO:  $bucket_info"
-  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
-  [[ $region != "" ]] || fail "empty bucket region"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}
-
 @test "test_retention_bypass" {
   test_retention_bypass_aws_root
 }
 
-@test "test_head_bucket_doesnt_exist" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
-  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
-  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}
-
 @test "test_add_object_metadata" {
   object_one="object-one"
   test_key="x-test-data"

tests/test_s3api_policy.sh

@@ -182,7 +182,8 @@ test_s3api_policy_invalid_action() {
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
 
   # shellcheck disable=SC2154
-  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
+  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
+  assert_success
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
@@ -190,13 +191,8 @@ test_s3api_policy_invalid_action() {
   run check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"; then
-    fail "put succeeded despite malformed policy"
-  fi
-  # shellcheck disable=SC2154
-  [[ "$put_bucket_policy_error" == *"MalformedPolicy"*"invalid action"* ]] || fail "invalid policy error: $put_bucket_policy_error"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file"
+  run put_and_check_for_malformed_policy "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
+  assert_success
 }
 
 test_s3api_policy_get_object_with_user() {
@@ -214,30 +210,26 @@ test_s3api_policy_get_object_with_user() {
   action="s3:GetObject"
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
 
-  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
-
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object"
-
-  if ! check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"; then
-    delete_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error deleting policy"
-    check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty after deletion"
-  fi
-
-  setup_user "$username" "$password" "user" || fail "error creating user"
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"; then
-    fail "get object with user succeeded despite lack of permissions"
-  fi
-  # shellcheck disable=SC2154
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
-
-  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
-  run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  run setup_user "$username" "$password" "user"
+  assert_success
+
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
+  assert_success
+
+  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
+  assert_success
+
+  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
+  assert_success
+
+  run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
+  assert_success
 }
 
 test_s3api_policy_get_object_specific_file() {
@@ -269,12 +261,8 @@ test_s3api_policy_get_object_specific_file() {
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
   assert_success
 
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"; then
-    fail "get object with user succeeded despite lack of permissions"
-  fi
-  # shellcheck disable=SC2154
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"
+  assert_success
 }
 
 test_s3api_policy_get_object_file_wildcard() {
@@ -292,17 +280,23 @@ test_s3api_policy_get_object_file_wildcard() {
   action="s3:GetObject"
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*"
 
-  setup_user "$username" "$password" "user" || fail "error creating user account"
+  run setup_user "$username" "$password" "user"
+  assert_success
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
-  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
+  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
+  assert_success
+  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
+  assert_success
 
-  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file" || fail "error copying object one"
-  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two" || fail "error copying object two"
-  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_three" "$BUCKET_ONE_NAME" "$policy_file_three" || fail "error copying object three"
+  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file"
+  assert_success
+  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two"
+  assert_success
+  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_three" "$BUCKET_ONE_NAME" "$policy_file_three"
+  assert_success
 
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file" "$TEST_FILE_FOLDER/$policy_file-copy" "$username" "$password"
   assert_success
@@ -310,12 +304,8 @@ test_s3api_policy_get_object_file_wildcard() {
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two" "$TEST_FILE_FOLDER/$policy_file_two-copy" "$username" "$password"
   assert_success
 
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$policy_file_three" "$TEST_FILE_FOLDER/$policy_file_three" "$username" "$password"; then
-    fail "get object three with user succeeded despite lack of permissions"
-  fi
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
-
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$policy_file_three" "$TEST_FILE_FOLDER/$policy_file_three" "$username" "$password"
+  assert_success
 }
 
 test_s3api_policy_get_object_folder_wildcard() {
@@ -364,25 +354,25 @@ test_s3api_policy_allow_deny() {
   run create_test_files "$policy_file" "$test_file"
   assert_success
 
-  setup_user "$username" "$password" "user" || fail "error creating user"
+  run setup_user "$username" "$password" "user"
+  assert_success
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
+  run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
     "Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" \
     "Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
+  assert_success
 
-  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
-  put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object to bucket"
+  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
+  assert_success
 
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"; then
-    fail "able to get object despite deny statement"
-  fi
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
 
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_file" "$test_file-copy" "$policy_file"
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
+  assert_success
 }
 
 test_s3api_policy_deny() {
@@ -409,12 +399,9 @@ test_s3api_policy_deny() {
   put_object "s3api" "$TEST_FILE_FOLDER/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one" || fail "error copying object one"
   put_object "s3api" "$TEST_FILE_FOLDER/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_two" || fail "error copying object two"
   get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_one" "$TEST_FILE_FOLDER/$test_file_one-copy" "$username" "$password" || fail "error getting object"
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"; then
-    fail "able to get object despite deny statement"
-  fi
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_file_one" "$test_file_two" "$test_file_one-copy" "$test_file_two-copy" "$policy_file"
+
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"
+  assert_success
 }
 
 test_s3api_policy_put_wildcard() {
@@ -447,13 +434,11 @@ test_s3api_policy_put_wildcard() {
   # shellcheck disable=SC2154
   [[ "$put_object_error" == *"Access Denied"* ]] || fail "invalid put object error: $put_object_error"
   put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password" || fail "error putting file despite policy permissions"
-  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"; then
-    fail "able to get object without permissions"
-  fi
-  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+
+  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"
+  assert_success
+
   download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" || fail "files don't match"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file"
 }
 
 test_s3api_policy_delete() {
@@ -488,8 +473,6 @@ test_s3api_policy_delete() {
   # shellcheck disable=SC2154
   [[ "$delete_object_error" == *"Access Denied"* ]] || fail "invalid delete object error: $delete_object_error"
   delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$username" "$password" || fail "error deleting object despite permissions"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_file_one" "$test_file_two" "$policy_file"
 }
 
 test_s3api_policy_get_bucket_policy() {
@@ -522,8 +505,6 @@ test_s3api_policy_get_bucket_policy() {
   log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file")"
   log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")"
   compare_files "$TEST_FILE_FOLDER/$policy_file" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file" "$policy_file-copy"
 }
 
 test_s3api_policy_list_multipart_uploads() {
@@ -567,8 +548,6 @@ test_s3api_policy_list_multipart_uploads() {
   log 5 "$uploads"
   upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1) || fail "error parsing upload key from uploads message: $upload_key"
   [[ $upload_key == "$test_file" ]] || fail "upload key doesn't match file marked as being uploaded"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file" "$test_file"
 }
 
 test_s3api_policy_put_bucket_policy() {
@@ -604,8 +583,6 @@ test_s3api_policy_put_bucket_policy() {
   log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file_two")"
   log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")"
   compare_files "$TEST_FILE_FOLDER/$policy_file_two" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file" "$policy_file_two" "$policy_file-copy"
 }
 
 test_s3api_policy_delete_bucket_policy() {
@@ -632,8 +609,6 @@ test_s3api_policy_delete_bucket_policy() {
   setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
   put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
   delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "unable to delete bucket policy"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_get_bucket_acl() {
@@ -706,9 +681,6 @@ test_s3api_policy_abort_multipart_upload() {
 
   put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
   abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password" || fail "error aborting multipart upload despite permissions"
-
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file" "$test_file"
 }
 
 test_s3api_policy_two_principals() {
@@ -741,9 +713,6 @@ test_s3api_policy_two_principals() {
   assert_success "error getting object with user $USERNAME_ONE"
   run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
   assert_success "error getting object with user $USERNAME_TWO"
-
-  delete_test_files "$test_file" "$policy_file" "$TEST_FILE_FOLDER/copy_one" "$TEST_FILE_FOLDER/copy_two"
-  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_put_bucket_tagging() {
@@ -767,9 +736,8 @@ test_s3api_policy_put_bucket_tagging() {
   run put_bucket_tagging_with_user "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" "$USERNAME_ONE" "$PASSWORD_ONE"
   assert_success "unable to put bucket tagging despite user permissions"
 
-  get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
-
-  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
+  run get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
+  assert_success
 }
 
 test_s3api_policy_put_acl() {
@@ -812,8 +780,6 @@ test_s3api_policy_put_acl() {
     id=$(echo "$second_grantee" | jq -r ".ID" 2>&1) || fail "error getting ID: $id"
     [[ $id == "all-users" ]] || fail "unexpected ID: $id"
   fi
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_get_bucket_tagging() {
@@ -842,11 +808,9 @@ test_s3api_policy_get_bucket_tagging() {
 
   run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
   assert_success "error putting policy"
+
   run get_and_check_bucket_tags_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
   assert_success "get and check bucket tags failed"
-
-  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_list_upload_parts() {
@@ -875,7 +839,4 @@ test_s3api_policy_list_upload_parts() {
 
   run create_upload_and_test_parts_listing "$test_file" "$policy_file"
   assert_success "error creating upload and testing parts listing"
-
-  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
-  delete_test_files "$policy_file" "$test_file"
 }

tests/test_s3cmd.sh

@@ -109,10 +109,6 @@ export RUN_USERS=true
 #  test_common_presigned_url_utf8_chars "s3cmd"
 #}
 
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3cmd"
-}
-
 @test "test_get_bucket_info_s3cmd" {
   run setup_bucket "s3cmd" "$BUCKET_ONE_NAME"
   assert_success

tests/test_s3cmd_file_count.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/test_common.sh
+
+export RUN_S3CMD=true
+
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}

tests/test_user_common.sh

@@ -70,7 +70,9 @@ test_create_user_already_exists() {
   username="$USERNAME_ONE"
   password="$PASSWORD_ONE"
 
-  setup_user "$username" "123456" "admin" || fail "error setting up user"
+  run setup_user "$username" "123456" "admin"
+  assert_success "error setting up user"
+
   if create_user "$username" "123456" "admin"; then
     fail "'user already exists' error not returned"
   fi

tests/util_list_objects.sh

@@ -166,3 +166,35 @@ list_objects_check_file_count() {
   fi
   return 0
 }
+
+check_object_listing_with_prefixes() {
+  if [ $# -ne 3 ]; then
+    log 2 "'check_object_listing_with_prefixes' requires bucket name, folder name, object name"
+    return 1
+  fi
+  if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"; then
+    log 2 "error listing objects with delimiter '/'"
+    return 1
+  fi
+  if ! prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then
+    log 2 "error getting object prefix from object list: $prefix"
+    return 1
+  fi
+  if [[ $prefix != "$2/" ]]; then
+    log 2 "prefix doesn't match (expected $2, actual $prefix/)"
+    return 1
+  fi
+  if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"; then
+    log 2 "error listing objects with delimiter '#"
+    return 1
+  fi
+  if ! key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1); then
+    log 2 "error getting key from object list: $key"
+    return 1
+  fi
+  if [[ $key != "$2/$3" ]]; then
+    log 2 "key doesn't match (expected $key, actual $2/$3)"
+    return 1
+  fi
+  return 0
+}

tests/util_policy.sh

@@ -205,3 +205,20 @@ get_and_check_policy() {
   fi
   return 0
 }
+
+put_and_check_for_malformed_policy() {
+  if [ $# -ne 2 ]; then
+    log 2 "'put_and_check_for_malformed_policy' requires bucket name, policy file"
+    return 1
+  fi
+  if put_bucket_policy "s3api" "$1" "$2"; then
+    log 2 "put succeeded despite malformed policy"
+    return 1
+  fi
+  # shellcheck disable=SC2154
+  if [[ "$put_bucket_policy_error" != *"MalformedPolicy"*"invalid action"* ]]; then
+    log 2 "invalid policy error: $put_bucket_policy_error"
+    return 1
+  fi
+  return 0
+}

tests/util_tags.sh

@@ -253,3 +253,60 @@ get_and_verify_object_tags() {
   fi
   return 0
 }
+
+verify_no_bucket_tags_rest() {
+  if [ $# -ne 1 ]; then
+    log 2 "'verify_no_bucket_tags_rest' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/bucket_tagging.txt" ./tests/rest_scripts/get_bucket_tagging.sh); then
+    log 2 "error listing bucket tags: $result"
+    return 1
+  fi
+  if [ "$result" != "404" ]; then
+    log 2 "expected response code of '404', was '$result' (error: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt"))"
+    return 1
+  fi
+  return 0
+}
+
+add_verify_bucket_tags_rest() {
+  if [ $# -ne 3 ]; then
+    log 2 "'add_verify_bucket_tags_rest' requires bucket name, test key, test value"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" TAG_KEY="$2" TAG_VALUE="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_tagging.sh); then
+    log 2 "error putting bucket tags: $result"
+    return 1
+  fi
+  if [ "$result" != "204" ]; then
+    log 2 "expected response code of '204', was '$result' (error: $(cat "$TEST_FILE_FOLDER/result.txt"))"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/bucket_tagging.txt" ./tests/rest_scripts/get_bucket_tagging.sh); then
+    log 2 "error listing bucket tags: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt"))"
+    return 1
+  fi
+  log 5 "tags: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt")"
+  if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/bucket_tagging.txt" 2>&1); then
+    log 2 "error retrieving key: $key"
+    return 1
+  fi
+  if [ "$key" != "$2" ]; then
+    log 2 "key mismatch (expected '$2', actual '$key')"
+    return 1
+  fi
+  if ! value=$(xmllint --xpath '//*[local-name()="Value"]/text()' "$TEST_FILE_FOLDER/bucket_tagging.txt" 2>&1); then
+    log 2 "error retrieving value: $value"
+    return 1
+  fi
+  if [ "$value" != "$3" ]; then
+    log 2 "value mismatch (expected '$3', actual '$value')"
+    return 1
+  fi
+  return 0
+}

tests/util_users.sh

@@ -399,4 +399,21 @@ get_bucket_owner() {
   log 3 "bucket owner for bucket '$1' not found"
   bucket_owner=
   return 0
+}
+
+verify_user_cant_get_object() {
+  if [ $# -ne 6 ]; then
+    log 2 "'verify_user_cant_get_object' requires client, bucket, key, save file, username, password"
+    return 1
+  fi
+  if get_object_with_user "$1" "$2" "$3" "$4" "$5" "$6"; then
+    log 2 "get object with user succeeded despite lack of permissions"
+    return 1
+  fi
+  # shellcheck disable=SC2154
+  if [[ "$get_object_error" != *"Access Denied"* ]]; then
+    log 2 "invalid get object error: $get_object_error"
+    return 1
+  fi
+  return 0
 }