fix: Fixes object version creatiion racing in posix

.github/workflows/betteralign.yml

@@ -1,23 +0,0 @@
-name: betteralign
-on: pull_request
-jobs:
-  build:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: "stable"
-        id: go
-
-      - name: Install betteralign
-        run: go install github.com/dkorunic/betteralign/cmd/betteralign@latest
-
-      - name: Run betteralign
-        run: betteralign -test_files ./...

.github/workflows/system.yml

@@ -8,96 +8,138 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - set: "mc, posix, non-file count, non-static, folder IAM"
+          - set: "s3cmd, posix"
+            LOCAL_FOLDER: /tmp/gw1
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
             IAM_TYPE: folder
-            RUN_SET: "mc-non-file-count"
+            USERS_FOLDER: /tmp/iam1
+            AWS_ENDPOINT_URL: https://127.0.0.1:7070
+            RUN_SET: "s3cmd"
             RECREATE_BUCKETS: "true"
+            PORT: 7070
             BACKEND: "posix"
-          - set: "mc, posix, file count, non-static, folder IAM"
+          - set: "s3, posix"
+            LOCAL_FOLDER: /tmp/gw2
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
             IAM_TYPE: folder
-            RUN_SET: "mc-file-count"
+            USERS_FOLDER: /tmp/iam2
+            AWS_ENDPOINT_URL: https://127.0.0.1:7071
+            RUN_SET: "s3"
             RECREATE_BUCKETS: "true"
+            PORT: 7071
             BACKEND: "posix"
-          - set: "REST, posix, non-static, all, folder IAM"
+          - set: "s3api non-policy, posix"
+            LOCAL_FOLDER: /tmp/gw3
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam3
+            AWS_ENDPOINT_URL: https://127.0.0.1:7072
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7072
+            BACKEND: "posix"
+          - set: "mc, posix"
+            LOCAL_FOLDER: /tmp/gw4
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam4
+            AWS_ENDPOINT_URL: https://127.0.0.1:7073
+            RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
+            PORT: 7073
+            BACKEND: "posix"
+          - set: "s3api-user, posix, s3 IAM"
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
+            IAM_TYPE: s3
+            USERS_BUCKET: versity-gwtest-iam
+            AWS_ENDPOINT_URL: https://127.0.0.1:7074
+            RUN_SET: "s3api-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7074
+            BACKEND: "posix"
+          - set: "s3api non-policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "s3api-non-policy"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: "s3api non-policy, s3 backend"
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
+          - set: "REST, posix"
+            LOCAL_FOLDER: /tmp/gw8
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam8
+            AWS_ENDPOINT_URL: https://127.0.0.1:7077
             RUN_SET: "rest"
             RECREATE_BUCKETS: "true"
+            PORT: 7077
             BACKEND: "posix"
-          - set: "s3, posix, non-file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-non-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-policy"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, user, non-static, s3 IAM"
-            IAM_TYPE: s3
-            RUN_SET: "s3api-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3api, posix, bucket, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, multipart, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-multipart"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, object, static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-object"
-            RECREATE_BUCKETS: "false"
-            BACKEND: "posix"
-          - set: "s3api, posix, policy, static, folder IAM"
+          - set: "s3api policy, static buckets"
+            LOCAL_FOLDER: /tmp/gw9
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-8
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-8
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam9
+            AWS_ENDPOINT_URL: https://127.0.0.1:7078
             RUN_SET: "s3api-policy"
             RECREATE_BUCKETS: "false"
+            PORT: 7078
             BACKEND: "posix"
-          - set: "s3api, posix, user, static, folder IAM"
+          - set: "s3api user, static buckets"
+            LOCAL_FOLDER: /tmp/gw10
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-9
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-9
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam10
+            AWS_ENDPOINT_URL: https://127.0.0.1:7079
             RUN_SET: "s3api-user"
             RECREATE_BUCKETS: "false"
+            PORT: 7079
             BACKEND: "posix"
-          - set: "s3api, s3, multipart|object, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "s3"
-          - set: "s3api, s3, policy|user, non-static, folder IAM"
+          - set: "s3api policy and user, posix"
+            LOCAL_FOLDER: /tmp/gw11
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-10
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-10
             IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam11
+            AWS_ENDPOINT_URL: https://127.0.0.1:7080
             RUN_SET: "s3api-policy,s3api-user"
             RECREATE_BUCKETS: "true"
+            PORT: 7080
+            BACKEND: "posix"
+          - set: "s3api policy and user, s3 backend"
+            LOCAL_FOLDER: /tmp/gw12
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-11
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-11
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam12
+            AWS_ENDPOINT_URL: https://127.0.0.1:7081
+            RUN_SET: "s3api-policy,s3api-user"
+            RECREATE_BUCKETS: "true"
+            PORT: 7081
             BACKEND: "s3"
-          - set: "s3cmd, posix, file count, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-file-count"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, non-user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-non-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
-          - set: "s3cmd, posix, user, non-static, folder IAM"
-            IAM_TYPE: folder
-            RUN_SET: "s3cmd-user"
-            RECREATE_BUCKETS: "true"
-            BACKEND: "posix"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -134,8 +176,15 @@ jobs:
 
       - name: Build and run
         env:
+          LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
+          BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
+          BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
+          USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
+          USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
           IAM_TYPE: ${{ matrix.IAM_TYPE }}
+          AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
           RUN_SET: ${{ matrix.RUN_SET }}
+          PORT: ${{ matrix.PORT }}
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
@@ -143,13 +192,6 @@ jobs:
           RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
-          LOCAL_FOLDER: /tmp/gw
-          BUCKET_ONE_NAME: versity-gwtest-bucket-one
-          BUCKET_TWO_NAME: versity-gwtest-bucket-two
-          USERS_FOLDER: /tmp/iam
-          USERS_BUCKET: versity-gwtest-iam
-          AWS_ENDPOINT_URL: https://127.0.0.1:7070
-          PORT: 7070
           S3CMD_CONFIG: tests/s3cfg.local.default
           MC_ALIAS: versity
           LOG_LEVEL: 4
@@ -162,7 +204,6 @@ jobs:
           REMOVE_TEST_FILE_FOLDER: true
           VERSIONING_DIR: ${{ github.workspace }}/versioning
           COMMAND_LOG: command.log
-          TIME_LOG: time.log
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
@@ -183,9 +224,6 @@ jobs:
           fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
-      - name: Time report
-        run: cat ${{ github.workspace }}/time.log
-
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

auth/acl.go

@@ -45,18 +45,18 @@ type GetBucketAclOutput struct {
 
 type PutBucketAclInput struct {
 	Bucket              *string
+	ACL                 types.BucketCannedACL
 	AccessControlPolicy *AccessControlPolicy
 	GrantFullControl    *string
 	GrantRead           *string
 	GrantReadACP        *string
 	GrantWrite          *string
 	GrantWriteACP       *string
-	ACL                 types.BucketCannedACL
 }
 
 type AccessControlPolicy struct {
-	Owner             *types.Owner
 	AccessControlList AccessControlList `xml:"AccessControlList"`
+	Owner             *types.Owner
 }
 
 type AccessControlList struct {
@@ -352,13 +352,13 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
 }
 
 type AccessOptions struct {
+	Acl           ACL
 	AclPermission types.Permission
+	IsRoot        bool
+	Acc           Account
 	Bucket        string
 	Object        string
 	Action        Action
-	Acl           ACL
-	Acc           Account
-	IsRoot        bool
 	Readonly      bool
 }
 

auth/bucket_policy.go

@@ -63,10 +63,10 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
 }
 
 type BucketPolicyItem struct {
+	Effect     BucketPolicyAccessType `json:"Effect"`
 	Principals Principals             `json:"Principal"`
 	Actions    Actions                `json:"Action"`
 	Resources  Resources              `json:"Resource"`
-	Effect     BucketPolicyAccessType `json:"Effect"`
 }
 
 func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {

auth/iam.go

@@ -93,6 +93,7 @@ var (
 )
 
 type Opts struct {
+	RootAccount            Account
 	Dir                    string
 	LDAPServerURL          string
 	LDAPBindDN             string
@@ -118,12 +119,11 @@ type Opts struct {
 	S3Region               string
 	S3Bucket               string
 	S3Endpoint             string
-	RootAccount            Account
-	CacheTTL               int
-	CachePrune             int
 	S3DisableSSlVerfiy     bool
 	S3Debug                bool
 	CacheDisable           bool
+	CacheTTL               int
+	CachePrune             int
 }
 
 func New(o *Opts) (IAMService, error) {

auth/iam_cache.go

@@ -36,14 +36,14 @@ type IAMCache struct {
 var _ IAMService = &IAMCache{}
 
 type item struct {
-	exp   time.Time
 	value Account
+	exp   time.Time
 }
 
 type icache struct {
-	items  map[string]item
-	expire time.Duration
 	sync.RWMutex
+	expire time.Duration
+	items  map[string]item
 }
 
 func (i *icache) set(k string, v Account) {

auth/iam_internal.go

@@ -33,8 +33,6 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
-	dir     string
-	rootAcc Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -42,6 +40,8 @@ type IAMServiceInternal struct {
 	// IAM service. All account updates should be sent to a single
 	// gateway instance if possible.
 	sync.RWMutex
+	dir     string
+	rootAcc Account
 }
 
 // UpdateAcctFunc accepts the current data and returns the new data to be stored

auth/iam_s3_object.go

@@ -42,14 +42,6 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
-	client *s3.Client
-
-	access   string
-	secret   string
-	region   string
-	bucket   string
-	endpoint string
-	rootAcc  Account
 	// This mutex will help with racing updates to the IAM data
 	// from multiple requests to this gateway instance, but
 	// will not help with racing updates to multiple load balanced
@@ -58,8 +50,15 @@ type IAMServiceS3 struct {
 	// gateway instance if possible.
 	sync.RWMutex
 
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
 	sslSkipVerify bool
 	debug         bool
+	rootAcc       Account
+	client        *s3.Client
 }
 
 var _ IAMService = &IAMServiceS3{}

auth/object_lock.go

@@ -29,9 +29,9 @@ import (
 )
 
 type BucketLockConfig struct {
+	Enabled          bool
 	DefaultRetention *types.DefaultRetention
 	CreatedAt        *time.Time
-	Enabled          bool
 }
 
 func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {

aws/signer/v4/v4.go

@@ -85,10 +85,6 @@ type keyDerivator interface {
 
 // SignerOptions is the SigV4 Signer options.
 type SignerOptions struct {
-
-	// The logger to send log messages to.
-	Logger logging.Logger
-
 	// Disables the Signer's moving HTTP header key/value pairs from the HTTP
 	// request header to the request's query string. This is most commonly used
 	// with pre-signed requests preventing headers from being added to the
@@ -104,6 +100,9 @@ type SignerOptions struct {
 	// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
 	DisableURIPathEscaping bool
 
+	// The logger to send log messages to.
+	Logger logging.Logger
+
 	// Enable logging of signed requests.
 	// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
 	// presigned URL.
@@ -118,8 +117,8 @@ type SignerOptions struct {
 // Signer applies AWS v4 signing to given request. Use this to sign requests
 // that need to be signed with AWS V4 Signatures.
 type Signer struct {
-	keyDerivator keyDerivator
 	options      SignerOptions
+	keyDerivator keyDerivator
 }
 
 // NewSigner returns a new SigV4 Signer
@@ -134,19 +133,17 @@ func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
 }
 
 type httpSigner struct {
-	KeyDerivator keyDerivator
 	Request      *http.Request
-	Credentials  aws.Credentials
-	Time         v4Internal.SigningTime
 	ServiceName  string
 	Region       string
+	Time         v4Internal.SigningTime
+	Credentials  aws.Credentials
+	KeyDerivator keyDerivator
+	IsPreSign    bool
+	SignedHdrs   []string
 
 	PayloadHash string
 
-	SignedHdrs []string
-
-	IsPreSign bool
-
 	DisableHeaderHoisting  bool
 	DisableURIPathEscaping bool
 	DisableSessionToken    bool

backend/posix/posix.go

@@ -29,6 +29,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -52,25 +54,27 @@ type Posix struct {
 	rootfd  *os.File
 	rootdir string
 
-	// bucket versioning directory path
-	versioningDir string
+	// chownuid/gid enable chowning of files to the account uid/gid
+	// when objects are uploaded
+	chownuid bool
+	chowngid bool
 
 	// euid/egid are the effective uid/gid of the running versitygw process
 	// used to determine if chowning is needed
 	euid int
 	egid int
 
-	// newDirPerm is the permission to set on newly created directories
-	newDirPerm fs.FileMode
-
-	// chownuid/gid enable chowning of files to the account uid/gid
-	// when objects are uploaded
-	chownuid bool
-	chowngid bool
-
 	// bucketlinks is a flag to enable symlinks to directories at the top
 	// level gateway directory to be treated as buckets the same as directories
 	bucketlinks bool
+
+	// bucket versioning directory path
+	versioningDir string
+
+	// newDirPerm is the permission to set on newly created directories
+	newDirPerm fs.FileMode
+
+	objLocks *sync.Map
 }
 
 var _ backend.Backend = &Posix{}
@@ -102,11 +106,11 @@ const (
 )
 
 type PosixOpts struct {
-	VersioningDir string
-	NewDirPerm    fs.FileMode
 	ChownUID      bool
 	ChownGID      bool
 	BucketLinks   bool
+	VersioningDir string
+	NewDirPerm    fs.FileMode
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -172,6 +176,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		bucketlinks:   opts.BucketLinks,
 		versioningDir: verioningdirAbs,
 		newDirPerm:    opts.NewDirPerm,
+		objLocks:      &sync.Map{},
 	}, nil
 }
 
@@ -2139,6 +2144,31 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	}, nil
 }
 
+type refCountedLock struct {
+	mu    sync.Mutex
+	count int32
+}
+
+func (p *Posix) getLock(key string) *refCountedLock {
+	actual, _ := p.objLocks.LoadOrStore(key, &refCountedLock{count: 0})
+	lock := actual.(*refCountedLock)
+
+	// Increment counter if the lock already exists
+	if actual != nil {
+		atomic.AddInt32(&lock.count, 1)
+	}
+	return lock
+}
+
+func (p *Posix) releaseLock(key string, lock *refCountedLock) {
+	lock.mu.Unlock()
+
+	// Decrement the counter
+	if atomic.AddInt32(&lock.count, -1) == 0 {
+		p.objLocks.Delete(key)
+	}
+}
+
 func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
 	acct, ok := ctx.Value("account").(auth.Account)
 	if !ok {
@@ -2228,13 +2258,20 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons
 	}
 	vEnabled := p.isBucketVersioningEnabled(vStatus)
 
+	// Lock the call, if it indicates a new object version creation
+	if vEnabled {
+		objLock := p.getLock(name)
+		objLock.mu.Lock()
+		defer p.releaseLock(name, objLock)
+	}
+
 	// object is file
 	d, err := os.Stat(name)
 	if err == nil && d.IsDir() {
 		return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
 	}
 
-	// if the versioninng is enabled first create the file object version
+	// if the versioning is enabled first create the file object version
 	if p.versioningEnabled() && vStatus != "" && err == nil {
 		var isVersionIdMissing bool
 		if p.isBucketVersioningSuspended(vStatus) {

backend/posix/with_otmpfile.go

@@ -38,12 +38,12 @@ type tmpfile struct {
 	f          *os.File
 	bucket     string
 	objname    string
+	isOTmp     bool
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	isOTmp     bool
-	needsChown bool
 }
 
 var (

backend/scoutfs/scoutfs.go

@@ -48,21 +48,12 @@ type ScoutfsOpts struct {
 }
 
 type ScoutFS struct {
-
-	// bucket/object metadata storage facility
-	meta meta.MetadataStorer
-
 	*posix.Posix
 	rootfd  *os.File
 	rootdir string
 
-	// euid/egid are the effective uid/gid of the running versitygw process
-	// used to determine if chowning is needed
-	euid int
-	egid int
-
-	// newDirPerm is the permissions to use when creating new directories
-	newDirPerm fs.FileMode
+	// bucket/object metadata storage facility
+	meta meta.MetadataStorer
 
 	// glaciermode enables the following behavior:
 	// GET object:  if file offline, return invalid object state
@@ -79,6 +70,14 @@ type ScoutFS struct {
 	// when objects are uploaded
 	chownuid bool
 	chowngid bool
+
+	// euid/egid are the effective uid/gid of the running versitygw process
+	// used to determine if chowning is needed
+	euid int
+	egid int
+
+	// newDirPerm is the permissions to use when creating new directories
+	newDirPerm fs.FileMode
 }
 
 var _ backend.Backend = &ScoutFS{}

backend/scoutfs/scoutfs_compat.go

@@ -70,10 +70,10 @@ type tmpfile struct {
 	bucket     string
 	objname    string
 	size       int64
+	needsChown bool
 	uid        int
 	gid        int
 	newDirPerm fs.FileMode
-	needsChown bool
 }
 
 var (

backend/walk.go

@@ -27,10 +27,10 @@ import (
 )
 
 type WalkResults struct {
-	NextMarker     string
 	CommonPrefixes []types.CommonPrefix
 	Objects        []s3response.Object
 	Truncated      bool
+	NextMarker     string
 }
 
 type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
@@ -268,18 +268,18 @@ func contains(a string, strs []string) bool {
 }
 
 type WalkVersioningResults struct {
-	NextMarker          string
-	NextVersionIdMarker string
 	CommonPrefixes      []types.CommonPrefix
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
 	Truncated           bool
+	NextMarker          string
+	NextVersionIdMarker string
 }
 
 type ObjVersionFuncResult struct {
-	NextVersionIdMarker string
 	ObjectVersions      []types.ObjectVersion
 	DelMarkers          []types.DeleteMarkerEntry
+	NextVersionIdMarker string
 	Truncated           bool
 }
 

backend/walk_test.go

@@ -41,8 +41,8 @@ type testcase struct {
 	prefix    string
 	delimiter string
 	marker    string
-	expected  backend.WalkResults
 	maxObjs   int32
+	expected  backend.WalkResults
 }
 
 func getObj(path string, d fs.DirEntry) (s3response.Object, error) {

metrics/metrics.go

@@ -43,14 +43,13 @@ type Tag struct {
 
 // Manager is a manager of metrics plugins
 type Manager struct {
+	wg  sync.WaitGroup
 	ctx context.Context
 
-	addDataChan chan datapoint
-
 	config Config
 
-	publishers []publisher
-	wg         sync.WaitGroup
+	publishers  []publisher
+	addDataChan chan datapoint
 }
 
 type Config struct {
@@ -221,6 +220,6 @@ func (m *Manager) addForwarder(addChan <-chan datapoint) {
 
 type datapoint struct {
 	key   string
-	tags  []Tag
 	value int64
+	tags  []Tag
 }

s3api/admin-server.go

@@ -26,11 +26,11 @@ import (
 )
 
 type S3AdminServer struct {
-	backend backend.Backend
 	app     *fiber.App
+	backend backend.Backend
 	router  *S3AdminRouter
-	cert    *tls.Certificate
 	port    string
+	cert    *tls.Certificate
 }
 
 func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {

s3api/controllers/admin_test.go

@@ -64,11 +64,11 @@ func TestAdminController_CreateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-create-user-malformed-body",
@@ -149,11 +149,11 @@ func TestAdminController_UpdateUser(t *testing.T) {
 	`
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-update-user-success",
@@ -223,11 +223,11 @@ func TestAdminController_DeleteUser(t *testing.T) {
 	app.Patch("/delete-user", adminController.DeleteUser)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-delete-user-success",
@@ -280,11 +280,11 @@ func TestAdminController_ListUsers(t *testing.T) {
 	appSucc.Patch("/list-users", adminController.ListUsers)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Admin-list-users-iam-error",
@@ -361,11 +361,11 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 	appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -424,11 +424,11 @@ func TestAdminController_ListBuckets(t *testing.T) {
 	app.Patch("/list-buckets", adminController.ListBuckets)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-buckets-success",

s3api/controllers/base.go

@@ -3308,14 +3308,14 @@ type MetaOpts struct {
 	Logger        s3log.AuditLogger
 	EvSender      s3event.S3EventSender
 	MetricsMng    *metrics.Manager
-	ObjectETag    *string
-	VersionId     *string
+	ContentLength int64
 	Action        string
 	BucketOwner   string
-	EventName     s3event.EventType
-	ContentLength int64
 	ObjectSize    int64
 	ObjectCount   int64
+	EventName     s3event.EventType
+	ObjectETag    *string
+	VersionId     *string
 	Status        int
 }
 

s3api/controllers/base_test.go

@@ -123,11 +123,11 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
 	appErr.Get("/", s3ApiControllerErr.ListBuckets)
 
 	tests := []struct {
+		name       string
 		args       args
 		app        *fiber.App
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "List-bucket-method-not-allowed",
@@ -233,11 +233,11 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-actions-get-tags-success",
@@ -435,11 +435,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
 	appError.Get("/:bucket", s3ApiControllerError.ListActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Get-bucket-tagging-non-existing-bucket",
@@ -728,11 +728,11 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
 	invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-bucket-tagging-invalid-body",
@@ -1034,11 +1034,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
 	invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Put-object-part-error-case",
@@ -1243,11 +1243,11 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
 	app.Delete("/:bucket", s3ApiController.DeleteBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-bucket-success",
@@ -1334,11 +1334,11 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
 	request.Header.Set("Content-Type", "application/xml")
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Delete-Objects-success",
@@ -1432,11 +1432,11 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
 	appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Abort-multipart-upload-success",
@@ -1541,11 +1541,11 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
 	appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-bucket-success",
@@ -1643,11 +1643,11 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Head-object-success",
@@ -1723,11 +1723,11 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 	app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
 
 	tests := []struct {
+		name       string
 		app        *fiber.App
 		args       args
-		name       string
-		statusCode int
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Restore-object-success",
@@ -1808,10 +1808,10 @@ func Test_XMLresponse(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",
@@ -1883,10 +1883,10 @@ func Test_response(t *testing.T) {
 	ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
 
 	tests := []struct {
-		args       args
 		name       string
-		statusCode int
+		args       args
 		wantErr    bool
+		statusCode int
 	}{
 		{
 			name: "Internal-server-error",

s3api/middlewares/authentication.go

@@ -149,8 +149,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 }
 
 type accounts struct {
-	iam  auth.IAMService
 	root RootUserConfig
+	iam  auth.IAMService
 }
 
 func (a accounts) getAccount(access string) (auth.Account, error) {

s3api/router_test.go

@@ -29,9 +29,9 @@ func TestS3ApiRouter_Init(t *testing.T) {
 		iam auth.IAMService
 	}
 	tests := []struct {
-		args args
-		sa   *S3ApiRouter
 		name string
+		sa   *S3ApiRouter
+		args args
 	}{
 		{
 			name: "Initialize S3 api router",

s3api/server.go

@@ -29,15 +29,15 @@ import (
 )
 
 type S3ApiServer struct {
-	backend  backend.Backend
 	app      *fiber.App
+	backend  backend.Backend
 	router   *S3ApiRouter
-	cert     *tls.Certificate
 	port     string
-	health   string
+	cert     *tls.Certificate
 	quiet    bool
 	debug    bool
 	readonly bool
+	health   string
 }
 
 func New(

s3api/server_test.go

@@ -39,9 +39,9 @@ func TestNew(t *testing.T) {
 	port := ":7070"
 
 	tests := []struct {
-		wantS3ApiServer *S3ApiServer
-		args            args
 		name            string
+		args            args
+		wantS3ApiServer *S3ApiServer
 		wantErr         bool
 	}{
 		{
@@ -78,8 +78,8 @@ func TestNew(t *testing.T) {
 
 func TestS3ApiServer_Serve(t *testing.T) {
 	tests := []struct {
-		sa      *S3ApiServer
 		name    string
+		sa      *S3ApiServer
 		wantErr bool
 	}{
 		{

s3api/utils/auth-reader.go

@@ -41,10 +41,10 @@ const (
 // the data is completely read.
 type AuthReader struct {
 	ctx    *fiber.Ctx
-	r      *HashReader
 	auth   AuthData
 	secret string
 	size   int
+	r      *HashReader
 	debug  bool
 }
 

s3api/utils/chunk-reader.go

@@ -48,15 +48,15 @@ const (
 // object data stream
 type ChunkReader struct {
 	r                io.Reader
-	chunkHash        hash.Hash
+	signingKey       []byte
 	prevSig          string
 	parsedSig        string
-	strToSignPrefix  string
-	signingKey       []byte
-	stash            []byte
 	currentChunkSize int64
 	chunkDataLeft    int64
 	trailerExpected  int
+	stash            []byte
+	chunkHash        hash.Hash
+	strToSignPrefix  string
 	skipcheck        bool
 }
 

s3api/utils/presign-auth-reader.go

@@ -41,10 +41,10 @@ const (
 // data requests where the data size is not known until
 // the data is completely read.
 type PresignedAuthReader struct {
-	r      io.Reader
 	ctx    *fiber.Ctx
 	auth   AuthData
 	secret string
+	r      io.Reader
 	debug  bool
 }
 

s3api/utils/presign-auth-reader_test.go

@@ -23,13 +23,13 @@ import (
 
 func Test_validateExpiration(t *testing.T) {
 	type args struct {
-		date time.Time
 		str  string
+		date time.Time
 	}
 	tests := []struct {
+		name string
 		args args
 		err  error
-		name string
 	}{
 		{
 			name: "empty-expiration",

s3api/utils/utils_test.go

@@ -49,11 +49,11 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
 
 	tests := []struct {
+		name    string
 		args    args
 		want    *http.Request
-		name    string
-		hdrs    []string
 		wantErr bool
+		hdrs    []string
 	}{
 		{
 			name: "Success-response",
@@ -101,9 +101,9 @@ func TestGetUserMetaData(t *testing.T) {
 	req := ctx.Request()
 
 	tests := []struct {
+		name         string
 		args         args
 		wantMetadata map[string]string
-		name         string
 	}{
 		{
 			name: "Success-empty-response",

s3event/event.go

@@ -30,11 +30,11 @@ type S3EventSender interface {
 }
 
 type EventMeta struct {
-	ObjectETag  *string
-	VersionId   *string
 	BucketOwner string
 	EventName   EventType
 	ObjectSize  int64
+	ObjectETag  *string
+	VersionId   *string
 }
 
 type EventSchema struct {
@@ -42,8 +42,6 @@ type EventSchema struct {
 }
 
 type EventRecord struct {
-	ResponseElements  EventResponseElements `json:"responseElements"`
-	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 	EventVersion      string                `json:"eventVersion"`
 	EventSource       string                `json:"eventSource"`
 	AwsRegion         string                `json:"awsRegion"`
@@ -51,7 +49,9 @@ type EventRecord struct {
 	EventName         EventType             `json:"eventName"`
 	UserIdentity      EventUserIdentity     `json:"userIdentity"`
 	RequestParameters EventRequestParams    `json:"requestParameters"`
+	ResponseElements  EventResponseElements `json:"responseElements"`
 	S3                EventS3Data           `json:"s3"`
+	GlacierEventData  EventGlacierData      `json:"glacierEventData"`
 }
 
 type EventUserIdentity struct {
@@ -99,11 +99,11 @@ type EventS3BucketData struct {
 }
 
 type EventObjectData struct {
+	Key       string  `json:"key"`
+	Size      int64   `json:"size"`
 	ETag      *string `json:"eTag"`
 	VersionId *string `json:"versionId"`
-	Key       string  `json:"key"`
 	Sequencer string  `json:"sequencer"`
-	Size      int64   `json:"size"`
 }
 
 type EventConfig struct {

s3event/kafka.go

@@ -31,9 +31,9 @@ import (
 var sequencer = 0
 
 type Kafka struct {
+	key    string
 	writer *kafka.Writer
 	filter EventFilter
-	key    string
 	mu     sync.Mutex
 }
 

s3event/nats.go

@@ -27,10 +27,10 @@ import (
 )
 
 type NatsEventSender struct {
-	client *nats.Conn
-	filter EventFilter
 	topic  string
+	client *nats.Conn
 	mu     sync.Mutex
+	filter EventFilter
 }
 
 func InitNatsEventService(url, topic string, filter EventFilter) (S3EventSender, error) {

s3event/webhook.go

@@ -30,9 +30,9 @@ import (
 )
 
 type Webhook struct {
+	url    string
 	client *http.Client
 	filter EventFilter
-	url    string
 	mu     sync.Mutex
 }
 

s3log/audit-logger.go

@@ -33,8 +33,8 @@ type AuditLogger interface {
 
 type LogMeta struct {
 	BucketOwner string
-	Action      string
 	ObjectSize  int64
+	Action      string
 	HttpStatus  int
 }
 
@@ -45,16 +45,21 @@ type LogConfig struct {
 }
 
 type LogFields struct {
-	Time               time.Time
 	BucketOwner        string
 	Bucket             string
+	Time               time.Time
 	RemoteIP           string
 	Requester          string
 	RequestID          string
 	Operation          string
 	Key                string
 	RequestURI         string
+	HttpStatus         int
 	ErrorCode          string
+	BytesSent          int
+	ObjectSize         int64
+	TotalTime          int64
+	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	VersionID          string
@@ -66,11 +71,6 @@ type LogFields struct {
 	TLSVersion         string
 	AccessPointARN     string
 	AclRequired        string
-	HttpStatus         int
-	BytesSent          int
-	ObjectSize         int64
-	TotalTime          int64
-	TurnAroundTime     int64
 }
 
 type AdminLogFields struct {
@@ -80,17 +80,17 @@ type AdminLogFields struct {
 	RequestID          string
 	Operation          string
 	RequestURI         string
+	HttpStatus         int
 	ErrorCode          string
+	BytesSent          int
+	TotalTime          int64
+	TurnAroundTime     int64
 	Referer            string
 	UserAgent          string
 	SignatureVersion   string
 	CipherSuite        string
 	AuthenticationType string
 	TLSVersion         string
-	HttpStatus         int
-	BytesSent          int
-	TotalTime          int64
-	TurnAroundTime     int64
 }
 
 type Loggers struct {

s3log/file.go

@@ -34,10 +34,10 @@ const (
 
 // FileLogger is a local file audit log
 type FileLogger struct {
-	f       *os.File
 	logfile string
-	mu      sync.Mutex
+	f       *os.File
 	gotErr  bool
+	mu      sync.Mutex
 }
 
 var _ AuditLogger = &FileLogger{}

s3log/webhook.go

@@ -33,8 +33,8 @@ import (
 
 // WebhookLogger is a webhook URL audit log
 type WebhookLogger struct {
-	url string
 	mu  sync.Mutex
+	url string
 }
 
 var _ AuditLogger = &WebhookLogger{}

s3response/s3response.go

@@ -35,17 +35,17 @@ type PutObjectOutput struct {
 
 // Part describes part metadata.
 type Part struct {
+	PartNumber   int
 	LastModified time.Time
 	ETag         string
-	PartNumber   int
 	Size         int64
 }
 
 func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Part
 	aux := &struct {
-		*Alias
 		LastModified string `xml:"LastModified"`
+		*Alias
 	}{
 		Alias: (*Alias)(&p),
 	}
@@ -59,23 +59,23 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 type ListPartsResult struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
 
-	Initiator Initiator
-	Owner     Owner
-
 	Bucket   string
 	Key      string
 	UploadID string `xml:"UploadId"`
 
+	Initiator Initiator
+	Owner     Owner
+
 	// The class of storage used to store the object.
 	StorageClass types.StorageClass
 
-	// List of parts.
-	Parts []Part `xml:"Part"`
-
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
 }
 
 type ObjectAttributes string
@@ -97,23 +97,23 @@ func (o ObjectAttributes) IsValid() bool {
 }
 
 type GetObjectAttributesResponse struct {
-	ETag        *string
-	ObjectSize  *int64
-	ObjectParts *ObjectParts
+	ETag         *string
+	ObjectSize   *int64
+	StorageClass types.StorageClass `xml:",omitempty"`
+	ObjectParts  *ObjectParts
 
 	// Not included in the response body
 	VersionId    *string
 	LastModified *time.Time
 	DeleteMarker *bool
-	StorageClass types.StorageClass `xml:",omitempty"`
 }
 
 type ObjectParts struct {
-	Parts                []types.ObjectPart `xml:"Part"`
 	PartNumberMarker     int
 	NextPartNumberMarker int
 	MaxParts             int
 	IsTruncated          bool
+	Parts                []types.ObjectPart `xml:"Part"`
 }
 
 // ListMultipartUploadsResponse - s3 api list multipart uploads response.
@@ -128,17 +128,18 @@ type ListMultipartUploadsResult struct {
 	Delimiter          string
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
+	MaxUploads         int
+	IsTruncated        bool
 
 	// List of pending uploads.
 	Uploads []Upload `xml:"Upload"`
 
 	// Delimed common prefixes.
 	CommonPrefixes []CommonPrefix
-	MaxUploads     int
-	IsTruncated    bool
 }
 
 type ListObjectsResult struct {
+	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name           *string
 	Prefix         *string
 	Marker         *string
@@ -146,13 +147,13 @@ type ListObjectsResult struct {
 	MaxKeys        *int32
 	Delimiter      *string
 	IsTruncated    *bool
-	XMLName        xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-	EncodingType   types.EncodingType
 	Contents       []Object
 	CommonPrefixes []types.CommonPrefix
+	EncodingType   types.EncodingType
 }
 
 type ListObjectsV2Result struct {
+	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
 	Name                  *string
 	Prefix                *string
 	StartAfter            *string
@@ -162,10 +163,9 @@ type ListObjectsV2Result struct {
 	MaxKeys               *int32
 	Delimiter             *string
 	IsTruncated           *bool
-	XMLName               xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-	EncodingType          types.EncodingType
 	Contents              []Object
 	CommonPrefixes        []types.CommonPrefix
+	EncodingType          types.EncodingType
 }
 
 type Object struct {
@@ -197,19 +197,19 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 // Upload describes in progress multipart upload
 type Upload struct {
-	Initiated    time.Time
-	Initiator    Initiator
-	Owner        Owner
 	Key          string
 	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
 	StorageClass types.StorageClass
+	Initiated    time.Time
 }
 
 func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias Upload
 	aux := &struct {
-		*Alias
 		Initiated string `xml:"Initiated"`
+		*Alias
 	}{
 		Alias: (*Alias)(&u),
 	}
@@ -261,11 +261,11 @@ type DeleteResult struct {
 }
 type SelectObjectContentPayload struct {
 	Expression          *string
+	ExpressionType      types.ExpressionType
 	RequestProgress     *types.RequestProgress
 	InputSerialization  *types.InputSerialization
 	OutputSerialization *types.OutputSerialization
 	ScanRange           *types.ScanRange
-	ExpressionType      types.ExpressionType
 }
 
 type SelectObjectContentResult struct {
@@ -283,30 +283,30 @@ type Bucket struct {
 
 type ListBucketsInput struct {
 	Owner             string
+	IsAdmin           bool
 	ContinuationToken string
 	Prefix            string
 	MaxBuckets        int32
-	IsAdmin           bool
 }
 
 type ListAllMyBucketsResult struct {
 	XMLName           xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
 	Owner             CanonicalUser
+	Buckets           ListAllMyBucketsList
 	ContinuationToken string `xml:"ContinuationToken,omitempty"`
 	Prefix            string `xml:"Prefix,omitempty"`
-	Buckets           ListAllMyBucketsList
 }
 
 type ListAllMyBucketsEntry struct {
-	CreationDate time.Time
 	Name         string
+	CreationDate time.Time
 }
 
 func (r ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias ListAllMyBucketsEntry
 	aux := &struct {
-		*Alias
 		CreationDate string `xml:"CreationDate"`
+		*Alias
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -335,8 +335,8 @@ type CopyObjectResult struct {
 func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type Alias CopyObjectResult
 	aux := &struct {
-		*Alias
 		LastModified string `xml:"LastModified"`
+		*Alias
 	}{
 		Alias: (*Alias)(&r),
 	}
@@ -404,15 +404,15 @@ type ListVersionsResult struct {
 }
 
 type GetBucketVersioningOutput struct {
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 	MFADelete *types.MFADeleteStatus
 	Status    *types.BucketVersioningStatus
-	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersioningConfiguration" json:"-"`
 }
 
 type PutObjectRetentionInput struct {
-	RetainUntilDate AmzDate
 	XMLName         xml.Name `xml:"Retention"`
 	Mode            types.ObjectLockRetentionMode
+	RetainUntilDate AmzDate
 }
 
 type AmzDate struct {

s3select/message-handler.go

@@ -204,6 +204,7 @@ func genErrorMessage(errorCode, errorMessage string) []byte {
 type GetProgress func() (bytesScanned int64, bytesProcessed int64)
 
 type MessageHandler struct {
+	sync.Mutex
 	ctx           context.Context
 	cancel        context.CancelFunc
 	writer        *bufio.Writer
@@ -212,7 +213,6 @@ type MessageHandler struct {
 	stopCh        chan bool
 	resetCh       chan bool
 	bytesReturned int64
-	sync.Mutex
 }
 
 // NewMessageHandler creates a new MessageHandler instance and starts the event streaming

tests/.env.default

@@ -28,5 +28,4 @@ PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
 VERSIONING_DIR=/tmp/versioning
-COMMAND_LOG=command.log
-TIME_LOG=time.log
+COMMAND_LOG=command.log

tests/docker-compose-bats.yml

@@ -27,7 +27,7 @@ services:
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
-    command: ["s3api-bucket,s3api-object"]
+    command: ["s3api-non-policy"]
   direct:
     build:
       dockerfile: tests/Dockerfile_direct

tests/integration/bench.go

@@ -26,9 +26,9 @@ import (
 )
 
 type prefResult struct {
-	err     error
 	elapsed time.Duration
 	size    int64
+	err     error
 }
 
 func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {

tests/integration/data-io.go

@@ -22,9 +22,9 @@ import (
 )
 
 type RReader struct {
-	hash     hash.Hash
 	buf      []byte
 	dataleft int
+	hash     hash.Hash
 }
 
 func NewDataReader(totalsize, bufsize int) *RReader {

tests/integration/group-tests.go

@@ -616,7 +616,7 @@ func TestVersioning(s *S3Conf) {
 	Versioning_WORM_obj_version_locked_with_governance_retention(s)
 	Versioning_WORM_obj_version_locked_with_compliance_retention(s)
 	// Concurrent requests
-	//Versioninig_concurrent_upload_object(s)
+	Versioning_concurrent_upload_object(s)
 }
 
 func TestVersioningDisabled(s *S3Conf) {

tests/integration/s3conf.go

@@ -35,10 +35,10 @@ type S3Conf struct {
 	awsSecret         string
 	awsRegion         string
 	endpoint          string
-	PartSize          int64
-	Concurrency       int
 	checksumDisable   bool
 	pathStyle         bool
+	PartSize          int64
+	Concurrency       int
 	debug             bool
 	versioningEnabled bool
 	azureTests        bool

tests/integration/tests.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -13386,34 +13387,43 @@ func Versioning_concurrent_upload_object(s *S3Conf) error {
 		versionCount := 5
 		// Channel to collect errors
 		errCh := make(chan error, versionCount)
+		// Channel to collect object verisons
+		versionsCh := make(chan types.ObjectVersion, versionCount)
 
-		uploadVersion := func(wg *sync.WaitGroup) {
+		uploadVersion := func(wg *sync.WaitGroup, length int64) {
 			defer wg.Done()
 
-			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-			res, err := s3client.PutObject(ctx, &s3.PutObjectInput{
+			out, err := putObjectWithData(length, &s3.PutObjectInput{
 				Bucket: &bucket,
 				Key:    &obj,
-			})
-			cancel()
+			}, s3client)
 			if err != nil {
 				// Send error to the channel
 				errCh <- err
-				return
 			}
 
-			fmt.Printf("uploaded object successfully: versionId: %v\n", *res.VersionId)
+			versionsCh <- types.ObjectVersion{
+				ETag:         out.res.ETag,
+				StorageClass: types.ObjectVersionStorageClassStandard,
+				IsLatest:     getBoolPtr(false),
+				Key:          &obj,
+				VersionId:    out.res.VersionId,
+				Size:         &length,
+			}
+
+			fmt.Printf("uploaded object successfully: versionId: %v\n", *out.res.VersionId)
 		}
 
 		wg := &sync.WaitGroup{}
 		wg.Add(versionCount)
 
 		for i := 0; i < versionCount; i++ {
-			go uploadVersion(wg)
+			go uploadVersion(wg, int64(i*100))
 		}
 
 		wg.Wait()
 		close(errCh)
+		close(versionsCh)
 
 		// Check if there were any errors
 		for err := range errCh {
@@ -13423,6 +13433,18 @@ func Versioning_concurrent_upload_object(s *S3Conf) error {
 			}
 		}
 
+		versions := []types.ObjectVersion{}
+
+		for el := range versionsCh {
+			versions = append(versions, el)
+		}
+
+		sort.SliceStable(versions, func(i, j int) bool {
+			return *versions[i].VersionId > *versions[j].VersionId
+		})
+
+		versions[0].IsLatest = getBoolPtr(true)
+
 		// List object versions after all uploads
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
 		res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
@@ -13433,8 +13455,8 @@ func Versioning_concurrent_upload_object(s *S3Conf) error {
 			return err
 		}
 
-		if len(res.Versions) != versionCount {
-			return fmt.Errorf("expected %v object versions, instead got %v", versionCount, len(res.Versions))
+		if !compareVersions(versions, res.Versions) {
+			return fmt.Errorf("expected the object versions to be %v, instead got %v", versions, res.Versions)
 		}
 
 		return nil

tests/integration/utils.go

@@ -168,9 +168,9 @@ func teardown(s *S3Conf, bucket string) error {
 }
 
 type setupCfg struct {
+	LockEnabled      bool
 	VersioningStatus types.BucketVersioningStatus
 	Ownership        types.ObjectOwnership
-	LockEnabled      bool
 }
 
 type setupOpt func(*setupCfg)
@@ -214,12 +214,12 @@ func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client,
 }
 
 type authConfig struct {
-	date     time.Time
 	testName string
 	path     string
 	method   string
-	service  string
 	body     []byte
+	service  string
+	date     time.Time
 }
 
 func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
@@ -434,9 +434,9 @@ func contains(s []string, e string) bool {
 }
 
 type putObjectOutput struct {
-	res  *s3.PutObjectOutput
-	data []byte
 	csum [32]byte
+	data []byte
+	res  *s3.PutObjectOutput
 }
 
 func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {

tests/rest_scripts/get_bucket_tagging.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/rest_scripts/rest.sh
-
-# Fields
-# shellcheck disable=SC2153
-bucket_name="$BUCKET_NAME"
-
-current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-
-canonical_request="GET
-/$bucket_name
-tagging=
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-create_canonical_hash_sts_and_signature
-
-curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?tagging="
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
--H "\"x-amz-date: $current_date_time\""
--o "$OUTPUT_FILE")
-# shellcheck disable=SC2154
-eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_bucket_tagging.sh

@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/rest_scripts/rest.sh
-
-# Fields
-
-# shellcheck disable=SC2153
-bucket_name="$BUCKET_NAME"
-# shellcheck disable=SC2153
-key="$TAG_KEY"
-# shellcheck disable=SC2153
-value="$TAG_VALUE"
-
-payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
-<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
-   <TagSet>
-      <Tag>
-         <Key>$key</Key>
-         <Value>$value</Value>
-      </Tag>
-   </TagSet>
-</Tagging>"
-
-content_md5=$(echo -n "$payload" | openssl dgst -binary -md5 | openssl base64)
-payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
-current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-
-canonical_request="PUT
-/$bucket_name
-tagging=
-content-md5:$content_md5
-host:$host
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-content-md5;host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-create_canonical_hash_sts_and_signature
-
-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?tagging="
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"Content-MD5: $content_md5\""
--H "\"x-amz-content-sha256: $payload_hash\""
--H "\"x-amz-date: $current_date_time\""
--d "\"${payload//\"/\\\"}\""
--o "$OUTPUT_FILE")
-
-# shellcheck disable=SC2154
-eval "${curl_command[*]}" 2>&1

tests/run.sh

@@ -19,21 +19,14 @@ show_help() {
     echo "Usage: $0 [option...]"
     echo "   -h, --help          Display this help message and exit"
     echo "                       Separate the below by comma"
-    echo "   s3api               Run all tests with s3api cli"
-    echo "   s3api-multipart     Run multipart tests with s3api cli"
-    echo "   s3api-bucket        Run bucket tests with s3api cli"
-    echo "   s3api-object        Run object tests with s3api cli"
+    echo "   s3api               Run tests with s3api cli"
+    echo "   s3api-non-policy    Run policy tests with s3api cli"
     echo "   s3api-policy        Run policy tests with s3api cli"
-    echo "   s3api-user          Run user tests with s3api cli"
     echo "   s3                  Run tests with s3 cli"
     echo "   s3cmd               Run tests with s3cmd utility"
-    echo "   s3cmd-user          Run user tests with s3cmd utility"
-    echo "   s3cmd-non-user      Run non-user tests with s3cmd utility"
-    echo "   s3cmd-file-count    Run file count test with s3cmd utility"
     echo "   mc                  Run tests with mc utility"
-    echo "   mc-non-file-count   Run non-file count tests with mc utility"
-    echo "   mc-file-count       Run file count test with mc utility"
     echo "   rest                Run tests with rest cli"
+    echo "   s3api-user          Run user tests with aws cli"
 }
 
 handle_param() {
@@ -42,7 +35,7 @@ handle_param() {
           show_help
           exit 0
           ;;
-      s3|s3-file-count|s3-non-file-count|s3api|s3cmd|s3cmd-user|s3cmd-non-user|s3cmd-file-count|mc|mc-non-file-count|mc-file-count|s3api-user|rest|s3api-policy|s3api-bucket|s3api-object|s3api-multipart)
+      s3|s3api|s3cmd|mc|s3api-user|rest|s3api-policy|s3api-non-policy)
           run_suite "$1"
           ;;
       *) # Handle unrecognized options or positional arguments
@@ -57,50 +50,25 @@ run_suite() {
   case $1 in
     s3api)
       echo "Running all s3api tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
-      fi
+      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
-      fi
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_aws.sh || exit_code=$?
       fi
       ;;
-    s3api-multipart)
-      echo "Running s3api multipart tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_multipart.sh || exit_code=$?
-      ;;
     s3api-policy)
       echo "Running s3api policy tests ..."
       "$HOME"/bin/bats ./tests/test_s3api_policy.sh || exit_code=$?
       ;;
-    s3api-bucket)
-      echo "Running s3api bucket tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_bucket.sh || exit_code=$?
-      ;;
-    s3api-object)
-      echo "Running s3api object tests ..."
-      "$HOME"/bin/bats ./tests/test_s3api_object.sh || exit_code=$?
+    s3api-non-policy)
+      echo "Running s3api non-policy tests ..."
+      "$HOME"/bin/bats ./tests/test_s3api.sh || exit_code=$?
       ;;
     s3)
       echo "Running s3 tests ..."
       "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
-      fi
-      ;;
-    s3-non-file-count)
-      echo "Running s3 non-file count tests ..."
-      "$HOME"/bin/bats ./tests/test_s3.sh || exit_code=$?
-      ;;
-    s3-file-count)
-      echo "Running s3 file count test ..."
-      "$HOME"/bin/bats ./tests/test_s3_file_count.sh || exit_code=$?
       ;;
     s3cmd)
       echo "Running s3cmd tests ..."
@@ -108,36 +76,10 @@ run_suite() {
       if [[ $exit_code -eq 0 ]]; then
         "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
       fi
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
-      fi
-      ;;
-    s3cmd-user)
-      echo "Running s3cmd user tests ..."
-      "$HOME"/bin/bats ./tests/test_user_s3cmd.sh || exit_code=$?
-      ;;
-    s3cmd-non-user)
-      echo "Running s3cmd non-user tests ..."
-      "$HOME"/bin/bats ./tests/test_s3cmd.sh || exit_code=$?
-      ;;
-    s3cmd-file-count)
-      echo "Running s3cmd file count test ..."
-      "$HOME"/bin/bats ./tests/test_s3cmd_file_count.sh || exit_code=$?
       ;;
     mc)
       echo "Running mc tests ..."
       "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
-      if [[ $exit_code -eq 0 ]]; then
-        "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
-      fi
-      ;;
-    mc-non-file-count)
-      echo "Running mc non-file count tests ..."
-      "$HOME"/bin/bats ./tests/test_mc.sh || exit_code=$?
-      ;;
-    mc-file-count)
-      echo "Running mc file count test ..."
-      "$HOME"/bin/bats ./tests/test_mc_file_count.sh || exit_code=$?
       ;;
     rest)
       echo "Running rest tests ..."

tests/setup.sh

@@ -28,7 +28,7 @@ setup() {
   base_setup
 
   log 4 "Running test $BATS_TEST_NAME"
-  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]]; then
     start_time=$(date +%s)
     export start_time
   fi
@@ -75,13 +75,9 @@ teardown() {
     fi
   fi
   stop_versity
-  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
+  if [[ $LOG_LEVEL -ge 5 ]]; then
     end_time=$(date +%s)
-    total_time=$((end_time - start_time))
-    log 4 "Total test time: $total_time"
-    if [[ -n "$TIME_LOG" ]]; then
-      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
-    fi
+    log 4 "Total test time: $((end_time - start_time))"
   fi
   if [[ -n "$COVERAGE_DB" ]]; then
     record_result

tests/test_mc.sh

@@ -112,6 +112,10 @@ export RUN_MC=true
   test_common_presigned_url_utf8_chars "mc"
 }
 
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "mc"
+}
+
 @test "test_create_bucket_invalid_name_mc" {
   if [[ $RECREATE_BUCKETS != "true" ]]; then
     return

tests/test_mc_file_count.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/test_common.sh
-
-export RUN_MC=true
-
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "mc"
-}

tests/test_rest.sh

@@ -373,25 +373,3 @@ source ./tests/util_versioning.sh
   run add_and_check_checksum "$TEST_FILE_FOLDER/$test_file" "$test_file"
   assert_success
 }
-
-@test "REST - bucket tagging - no tags" {
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run verify_no_bucket_tags_rest "$BUCKET_ONE_NAME"
-  assert_success
-}
-
-@test "REST - bucket tagging - tags" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/932"
-  fi
-  test_key="testKey"
-  test_value="testValue"
-
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run add_verify_bucket_tags_rest "$BUCKET_ONE_NAME" "$test_key" "$test_value"
-  assert_success
-}

tests/test_s3.sh

@@ -51,6 +51,10 @@ source ./tests/util_file.sh
   test_common_list_buckets "s3"
 }
 
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3"
+}
+
 @test "test_delete_bucket" {
   if [[ $RECREATE_BUCKETS == "false" ]]; then
     skip "will not test bucket deletion in static bucket test config"

tests/test_s3_file_count.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/test_common.sh
-source ./tests/util_file.sh
-
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3"
-}

tests/test_s3api.sh

@@ -20,6 +20,7 @@ source ./tests/util_aws.sh
 source ./tests/util_create_bucket.sh
 source ./tests/util_file.sh
 source ./tests/util_lock_config.sh
+source ./tests/util_multipart.sh
 source ./tests/util_tags.sh
 source ./tests/util_users.sh
 source ./tests/test_aws_root_inner.sh
@@ -37,6 +38,7 @@ source ./tests/commands/get_object_legal_hold.sh
 source ./tests/commands/get_object_lock_configuration.sh
 source ./tests/commands/get_object_retention.sh
 source ./tests/commands/get_object_tagging.sh
+source ./tests/commands/list_multipart_uploads.sh
 source ./tests/commands/list_object_versions.sh
 source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/put_bucket_policy.sh
@@ -50,6 +52,16 @@ source ./tests/commands/select_object_content.sh
 
 export RUN_USERS=true
 
+# abort-multipart-upload
+@test "test_abort_multipart_upload" {
+  test_abort_multipart_upload_aws_root
+}
+
+# complete-multipart-upload
+@test "test_complete_multipart_upload" {
+  test_complete_multipart_upload_aws_root
+}
+
 # copy-object
 @test "test_copy_object" {
   test_common_copy_object "s3api"
@@ -59,6 +71,35 @@ export RUN_USERS=true
   copy_object_empty || fail "copy objects with no parameters test failure"
 }
 
+# create-bucket
+@test "test_create_delete_bucket_aws" {
+  test_common_create_delete_bucket "aws"
+}
+
+@test "test_create_bucket_invalid_name" {
+  test_create_bucket_invalid_name_aws_root
+}
+
+# create-multipart-upload
+@test "test_create_multipart_upload_properties" {
+  test_create_multipart_upload_properties_aws_root
+}
+
+# delete-bucket - test_create_delete_bucket_aws
+
+# delete-bucket-policy
+@test "test_get_put_delete_bucket_policy" {
+  if [[ -n $SKIP_POLICY ]]; then
+    skip "will not test policy actions with SKIP_POLICY set"
+  fi
+  test_common_get_put_delete_bucket_policy "aws"
+}
+
+# delete-bucket-tagging
+@test "test-set-get-delete-bucket-tags" {
+  test_common_set_get_delete_bucket_tags "aws"
+}
+
 # delete-object - tested with bucket cleanup before or after tests
 
 # delete-object-tagging
@@ -74,6 +115,20 @@ export RUN_USERS=true
   test_delete_objects_aws_root
 }
 
+# get-bucket-acl
+@test "test_get_bucket_acl" {
+  test_get_bucket_acl_aws_root
+}
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "aws"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging - test_set_get_delete_bucket_tags
+
 # get-object
 @test "test_get_object_full_range" {
   test_get_object_full_range_aws_root
@@ -88,6 +143,12 @@ export RUN_USERS=true
   test_get_object_attributes_aws_root
 }
 
+@test "test_head_bucket_invalid_name" {
+  if head_bucket "aws" ""; then
+    fail "able to get bucket info for invalid name"
+  fi
+}
+
 @test "test_put_object" {
   test_put_object_aws_root
 }
@@ -107,6 +168,11 @@ export RUN_USERS=true
   test_common_put_object_no_data "aws"
 }
 
+# test listing buckets on versitygw
+@test "test_list_buckets" {
+  test_common_list_buckets "s3api"
+}
+
 # test listing a bucket's objects on versitygw
 @test "test_list_objects" {
   test_common_list_objects "aws"
@@ -120,6 +186,10 @@ export RUN_USERS=true
   test_get_put_object_retention_aws_root
 }
 
+@test "test_put_bucket_acl" {
+  test_common_put_bucket_acl "s3api"
+}
+
 # test v1 s3api list objects command
 @test "test-s3api-list-objects-v1" {
   test_s3api_list_objects_v1_aws_root
@@ -135,6 +205,77 @@ export RUN_USERS=true
   test_common_set_get_object_tags "aws"
 }
 
+# test multi-part upload list parts command
+@test "test-multipart-upload-list-parts" {
+  test_multipart_upload_list_parts_aws_root
+}
+
+# test listing of active uploads
+@test "test-multipart-upload-list-uploads" {
+  local bucket_file_one="bucket-file-one"
+  local bucket_file_two="bucket-file-two"
+
+  if [[ $RECREATE_BUCKETS == false ]]; then
+    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  run create_test_files "$bucket_file_one" "$bucket_file_two"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
+  assert_success
+}
+
+@test "test-multipart-upload-from-bucket" {
+  local bucket_file="bucket-file"
+
+  run create_test_file "$bucket_file"
+  assert_success
+
+  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
+  assert_success
+
+  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
+  assert_success
+
+  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_too_large" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
+  assert_success
+}
+
+@test "test_multipart_upload_from_bucket_range_valid" {
+  local bucket_file="bucket-file"
+  run create_large_file "$bucket_file"
+  assert_success
+
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
+  assert_success
+}
+
 @test "test-presigned-url-utf8-chars" {
   test_common_presigned_url_utf8_chars "aws"
 }
@@ -152,11 +293,18 @@ export RUN_USERS=true
   run setup_bucket "aws" "$BUCKET_ONE_NAME"
   assert_success
 
-  run put_object "aws" "$TEST_FILE_FOLDER/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name"
-  assert_success
+  put_object "aws" "$TEST_FILE_FOLDER/$folder_name/$object_name" "$BUCKET_ONE_NAME" "$folder_name/$object_name" || fail "failed to add object to bucket"
 
-  run check_object_listing_with_prefixes "$BUCKET_ONE_NAME" "$folder_name" "$object_name"
-  assert_success
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"
+  prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1) || fail "error getting object prefix from object list: $prefix"
+  [[ $prefix == "$folder_name/" ]] || fail "prefix doesn't match (expected $prefix, actual $folder_name/)"
+
+  list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"
+  key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1) || fail "error getting key from object list: $key"
+  [[ $key == "$folder_name/$object_name" ]] || fail "key doesn't match (expected $key, actual $folder_name/$object_name)"
+
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files $folder_name
 }
 
 # ensure that lists of files greater than a size of 1000 (pagination) are returned properly
@@ -183,10 +331,31 @@ export RUN_USERS=true
 #  [[ $put_object -eq 0 ]] || fail "Failed to add object to bucket"
 #}
 
+@test "test_head_bucket" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
+  log 5 "INFO:  $bucket_info"
+  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
+  [[ $region != "" ]] || fail "empty bucket region"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}
+
 @test "test_retention_bypass" {
   test_retention_bypass_aws_root
 }
 
+@test "test_head_bucket_doesnt_exist" {
+  run setup_bucket "aws" "$BUCKET_ONE_NAME"
+  assert_success
+
+  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
+  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
+  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+}
+
 @test "test_add_object_metadata" {
   object_one="object-one"
   test_key="x-test-data"

tests/test_s3api_bucket.sh

@@ -1,125 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/setup.sh
-source ./tests/util.sh
-source ./tests/util_aws.sh
-source ./tests/util_create_bucket.sh
-source ./tests/util_file.sh
-source ./tests/util_lock_config.sh
-source ./tests/util_tags.sh
-source ./tests/util_users.sh
-source ./tests/test_aws_root_inner.sh
-source ./tests/test_common.sh
-source ./tests/test_common_acl.sh
-source ./tests/commands/copy_object.sh
-source ./tests/commands/delete_bucket_policy.sh
-source ./tests/commands/delete_object_tagging.sh
-source ./tests/commands/get_bucket_acl.sh
-source ./tests/commands/get_bucket_policy.sh
-source ./tests/commands/get_bucket_versioning.sh
-source ./tests/commands/get_object.sh
-source ./tests/commands/get_object_attributes.sh
-source ./tests/commands/get_object_legal_hold.sh
-source ./tests/commands/get_object_lock_configuration.sh
-source ./tests/commands/get_object_retention.sh
-source ./tests/commands/get_object_tagging.sh
-source ./tests/commands/list_object_versions.sh
-source ./tests/commands/put_bucket_acl.sh
-source ./tests/commands/put_bucket_policy.sh
-source ./tests/commands/put_bucket_versioning.sh
-source ./tests/commands/put_object.sh
-source ./tests/commands/put_object_legal_hold.sh
-source ./tests/commands/put_object_lock_configuration.sh
-source ./tests/commands/put_object_retention.sh
-source ./tests/commands/put_public_access_block.sh
-source ./tests/commands/select_object_content.sh
-
-export RUN_USERS=true
-
-# create-bucket
-@test "test_create_delete_bucket_aws" {
-  test_common_create_delete_bucket "aws"
-}
-
-@test "test_create_bucket_invalid_name" {
-  test_create_bucket_invalid_name_aws_root
-}
-
-# delete-bucket - test_create_delete_bucket_aws
-
-# delete-bucket-policy
-@test "test_get_put_delete_bucket_policy" {
-  if [[ -n $SKIP_POLICY ]]; then
-    skip "will not test policy actions with SKIP_POLICY set"
-  fi
-  test_common_get_put_delete_bucket_policy "aws"
-}
-
-# delete-bucket-tagging
-@test "test-set-get-delete-bucket-tags" {
-  test_common_set_get_delete_bucket_tags "aws"
-}
-
-# get-bucket-acl
-@test "test_get_bucket_acl" {
-  test_get_bucket_acl_aws_root
-}
-
-# get-bucket-location
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "aws"
-}
-
-# get-bucket-policy - test_get_put_delete_bucket_policy
-
-# get-bucket-tagging - test_set_get_delete_bucket_tags
-
-@test "test_head_bucket_invalid_name" {
-  if head_bucket "aws" ""; then
-    fail "able to get bucket info for invalid name"
-  fi
-}
-
-# test listing buckets on versitygw
-@test "test_list_buckets" {
-  test_common_list_buckets "s3api"
-}
-
-@test "test_put_bucket_acl" {
-  test_common_put_bucket_acl "s3api"
-}
-
-@test "test_head_bucket" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME" || fail "error getting bucket info"
-  log 5 "INFO:  $bucket_info"
-  region=$(echo "$bucket_info" | grep -v "InsecureRequestWarning" | jq -r ".BucketRegion" 2>&1) || fail "error getting bucket region: $region"
-  [[ $region != "" ]] || fail "empty bucket region"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}
-
-@test "test_head_bucket_doesnt_exist" {
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  head_bucket "aws" "$BUCKET_ONE_NAME"a || local info_result=$?
-  [[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
-  [[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
-  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
-}

tests/test_s3api_multipart.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/setup.sh
-source ./tests/test_aws_root_inner.sh
-source ./tests/util_file.sh
-source ./tests/util_multipart.sh
-source ./tests/util_tags.sh
-source ./tests/commands/get_object.sh
-source ./tests/commands/put_object.sh
-source ./tests/commands/list_multipart_uploads.sh
-
-# abort-multipart-upload
-@test "test_abort_multipart_upload" {
-  test_abort_multipart_upload_aws_root
-}
-
-# complete-multipart-upload
-@test "test_complete_multipart_upload" {
-  test_complete_multipart_upload_aws_root
-}
-
-# create-multipart-upload
-@test "test_create_multipart_upload_properties" {
-  test_create_multipart_upload_properties_aws_root
-}
-
-# test multi-part upload list parts command
-@test "test-multipart-upload-list-parts" {
-  test_multipart_upload_list_parts_aws_root
-}
-
-# test listing of active uploads
-@test "test-multipart-upload-list-uploads" {
-  local bucket_file_one="bucket-file-one"
-  local bucket_file_two="bucket-file-two"
-
-  if [[ $RECREATE_BUCKETS == false ]]; then
-    run abort_all_multipart_uploads "$BUCKET_ONE_NAME"
-    assert_success
-  fi
-
-  run create_test_files "$bucket_file_one" "$bucket_file_two"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run create_list_check_multipart_uploads "$BUCKET_ONE_NAME" "$bucket_file_one" "$bucket_file_two"
-  assert_success
-}
-
-@test "test-multipart-upload-from-bucket" {
-  local bucket_file="bucket-file"
-
-  run create_test_file "$bucket_file"
-  assert_success
-
-  run dd if=/dev/urandom of="$TEST_FILE_FOLDER/$bucket_file" bs=5M count=1
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_from_bucket "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file" 4
-  assert_success
-
-  run get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file-copy" "$TEST_FILE_FOLDER/$bucket_file-copy"
-  assert_success
-
-  run compare_files "$TEST_FILE_FOLDER"/$bucket_file-copy "$TEST_FILE_FOLDER"/$bucket_file
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_too_large" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run multipart_upload_range_too_large "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER"/"$bucket_file"
-  assert_success
-}
-
-@test "test_multipart_upload_from_bucket_range_valid" {
-  local bucket_file="bucket-file"
-  run create_large_file "$bucket_file"
-  assert_success
-
-  run setup_bucket "aws" "$BUCKET_ONE_NAME"
-  assert_success
-
-  run run_and_verify_multipart_upload_with_valid_range "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file"
-  assert_success
-}

tests/test_s3api_policy.sh

@@ -182,8 +182,7 @@ test_s3api_policy_invalid_action() {
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
 
   # shellcheck disable=SC2154
-  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
-  assert_success
+  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
@@ -191,8 +190,13 @@ test_s3api_policy_invalid_action() {
   run check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  run put_and_check_for_malformed_policy "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
-  assert_success
+  if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"; then
+    fail "put succeeded despite malformed policy"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$put_bucket_policy_error" == *"MalformedPolicy"*"invalid action"* ]] || fail "invalid policy error: $put_bucket_policy_error"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_get_object_with_user() {
@@ -210,26 +214,30 @@ test_s3api_policy_get_object_with_user() {
   action="s3:GetObject"
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
 
+  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
+  put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object"
 
-  run setup_user "$username" "$password" "user"
-  assert_success
+  if ! check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"; then
+    delete_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error deleting policy"
+    check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty after deletion"
+  fi
 
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
-  assert_success
-
-  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource"
-  assert_success
-
-  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
-  assert_success
+  setup_user "$username" "$password" "user" || fail "error creating user"
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"; then
+    fail "get object with user succeeded despite lack of permissions"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
 
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
   assert_success
+
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_get_object_specific_file() {
@@ -261,8 +269,12 @@ test_s3api_policy_get_object_specific_file() {
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
   assert_success
 
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"
-  assert_success
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"; then
+    fail "get object with user succeeded despite lack of permissions"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_get_object_file_wildcard() {
@@ -280,23 +292,17 @@ test_s3api_policy_get_object_file_wildcard() {
   action="s3:GetObject"
   resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*"
 
-  run setup_user "$username" "$password" "user"
-  assert_success
+  setup_user "$username" "$password" "user" || fail "error creating user account"
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  run setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource"
-  assert_success
-  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
-  assert_success
+  setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
 
-  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file"
-  assert_success
-  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two"
-  assert_success
-  run put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_three" "$BUCKET_ONE_NAME" "$policy_file_three"
-  assert_success
+  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file" || fail "error copying object one"
+  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two" || fail "error copying object two"
+  put_object "s3api" "$TEST_FILE_FOLDER/$policy_file_three" "$BUCKET_ONE_NAME" "$policy_file_three" || fail "error copying object three"
 
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$policy_file" "$BUCKET_ONE_NAME" "$policy_file" "$TEST_FILE_FOLDER/$policy_file-copy" "$username" "$password"
   assert_success
@@ -304,8 +310,12 @@ test_s3api_policy_get_object_file_wildcard() {
   run download_and_compare_file_with_user "s3api" "$TEST_FILE_FOLDER/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two" "$TEST_FILE_FOLDER/$policy_file_two-copy" "$username" "$password"
   assert_success
 
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$policy_file_three" "$TEST_FILE_FOLDER/$policy_file_three" "$username" "$password"
-  assert_success
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$policy_file_three" "$TEST_FILE_FOLDER/$policy_file_three" "$username" "$password"; then
+    fail "get object three with user succeeded despite lack of permissions"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_get_object_folder_wildcard() {
@@ -354,25 +364,25 @@ test_s3api_policy_allow_deny() {
   run create_test_files "$policy_file" "$test_file"
   assert_success
 
-  run setup_user "$username" "$password" "user"
-  assert_success
+  setup_user "$username" "$password" "user" || fail "error creating user"
 
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 
-  run setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
+  setup_policy_with_double_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" \
     "Deny" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file" \
     "Allow" "$username" "s3:GetObject" "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
-  assert_success
 
-  run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
-  assert_success
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
+  put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object to bucket"
 
-  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"; then
+    fail "able to get object despite deny statement"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
 
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy" "$username" "$password"
-  assert_success
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file" "$test_file-copy" "$policy_file"
 }
 
 test_s3api_policy_deny() {
@@ -399,9 +409,12 @@ test_s3api_policy_deny() {
   put_object "s3api" "$TEST_FILE_FOLDER/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one" || fail "error copying object one"
   put_object "s3api" "$TEST_FILE_FOLDER/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_two" || fail "error copying object two"
   get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_one" "$TEST_FILE_FOLDER/$test_file_one-copy" "$username" "$password" || fail "error getting object"
-
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"
-  assert_success
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$TEST_FILE_FOLDER/$test_file_two-copy" "$username" "$password"; then
+    fail "able to get object despite deny statement"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file_one" "$test_file_two" "$test_file_one-copy" "$test_file_two-copy" "$policy_file"
 }
 
 test_s3api_policy_put_wildcard() {
@@ -434,11 +447,13 @@ test_s3api_policy_put_wildcard() {
   # shellcheck disable=SC2154
   [[ "$put_object_error" == *"Access Denied"* ]] || fail "invalid put object error: $put_object_error"
   put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password" || fail "error putting file despite policy permissions"
-
-  run verify_user_cant_get_object "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"
-  assert_success
-
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_folder/$test_file-copy" "$username" "$password"; then
+    fail "able to get object without permissions"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
   download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$TEST_FILE_FOLDER/$test_file-copy" || fail "files don't match"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file"
 }
 
 test_s3api_policy_delete() {
@@ -473,6 +488,8 @@ test_s3api_policy_delete() {
   # shellcheck disable=SC2154
   [[ "$delete_object_error" == *"Access Denied"* ]] || fail "invalid delete object error: $delete_object_error"
   delete_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$username" "$password" || fail "error deleting object despite permissions"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file_one" "$test_file_two" "$policy_file"
 }
 
 test_s3api_policy_get_bucket_policy() {
@@ -505,6 +522,8 @@ test_s3api_policy_get_bucket_policy() {
   log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file")"
   log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")"
   compare_files "$TEST_FILE_FOLDER/$policy_file" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file" "$policy_file-copy"
 }
 
 test_s3api_policy_list_multipart_uploads() {
@@ -548,6 +567,8 @@ test_s3api_policy_list_multipart_uploads() {
   log 5 "$uploads"
   upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1) || fail "error parsing upload key from uploads message: $upload_key"
   [[ $upload_key == "$test_file" ]] || fail "upload key doesn't match file marked as being uploaded"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file" "$test_file"
 }
 
 test_s3api_policy_put_bucket_policy() {
@@ -583,6 +604,8 @@ test_s3api_policy_put_bucket_policy() {
   log 5 "ORIG: $(cat "$TEST_FILE_FOLDER/$policy_file_two")"
   log 5 "COPY: $(cat "$TEST_FILE_FOLDER/$policy_file-copy")"
   compare_files "$TEST_FILE_FOLDER/$policy_file_two" "$TEST_FILE_FOLDER/$policy_file-copy" || fail "policies not equal"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file" "$policy_file_two" "$policy_file-copy"
 }
 
 test_s3api_policy_delete_bucket_policy() {
@@ -609,6 +632,8 @@ test_s3api_policy_delete_bucket_policy() {
   setup_policy_with_single_statement "$TEST_FILE_FOLDER/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
   put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
   delete_bucket_policy_with_user "$BUCKET_ONE_NAME" "$username" "$password" || fail "unable to delete bucket policy"
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_get_bucket_acl() {
@@ -681,6 +706,9 @@ test_s3api_policy_abort_multipart_upload() {
 
   put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file" || fail "error putting policy"
   abort_multipart_upload_with_user "$BUCKET_ONE_NAME" "$test_file" "$upload_id" "$username" "$password" || fail "error aborting multipart upload despite permissions"
+
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file" "$test_file"
 }
 
 test_s3api_policy_two_principals() {
@@ -713,6 +741,9 @@ test_s3api_policy_two_principals() {
   assert_success "error getting object with user $USERNAME_ONE"
   run get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/copy_two" "$USERNAME_TWO" "$PASSWORD_TWO"
   assert_success "error getting object with user $USERNAME_TWO"
+
+  delete_test_files "$test_file" "$policy_file" "$TEST_FILE_FOLDER/copy_one" "$TEST_FILE_FOLDER/copy_two"
+  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_put_bucket_tagging() {
@@ -736,8 +767,9 @@ test_s3api_policy_put_bucket_tagging() {
   run put_bucket_tagging_with_user "$BUCKET_ONE_NAME" "$tag_key" "$tag_value" "$USERNAME_ONE" "$PASSWORD_ONE"
   assert_success "unable to put bucket tagging despite user permissions"
 
-  run get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
-  assert_success
+  get_and_check_bucket_tags "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
+
+  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
 }
 
 test_s3api_policy_put_acl() {
@@ -780,6 +812,8 @@ test_s3api_policy_put_acl() {
     id=$(echo "$second_grantee" | jq -r ".ID" 2>&1) || fail "error getting ID: $id"
     [[ $id == "all-users" ]] || fail "unexpected ID: $id"
   fi
+  bucket_cleanup "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_get_bucket_tagging() {
@@ -808,9 +842,11 @@ test_s3api_policy_get_bucket_tagging() {
 
   run put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$policy_file"
   assert_success "error putting policy"
-
   run get_and_check_bucket_tags_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$tag_key" "$tag_value"
   assert_success "get and check bucket tags failed"
+
+  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file"
 }
 
 test_s3api_policy_list_upload_parts() {
@@ -839,4 +875,7 @@ test_s3api_policy_list_upload_parts() {
 
   run create_upload_and_test_parts_listing "$test_file" "$policy_file"
   assert_success "error creating upload and testing parts listing"
+
+  bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file" "$test_file"
 }

tests/test_s3cmd.sh

@@ -109,6 +109,10 @@ export RUN_USERS=true
 #  test_common_presigned_url_utf8_chars "s3cmd"
 #}
 
+@test "test_list_objects_file_count" {
+  test_common_list_objects_file_count "s3cmd"
+}
+
 @test "test_get_bucket_info_s3cmd" {
   run setup_bucket "s3cmd" "$BUCKET_ONE_NAME"
   assert_success

tests/test_s3cmd_file_count.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bats
-
-# Copyright 2024 Versity Software
-# This file is licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source ./tests/test_common.sh
-
-export RUN_S3CMD=true
-
-@test "test_list_objects_file_count" {
-  test_common_list_objects_file_count "s3cmd"
-}

tests/test_user_common.sh

@@ -70,9 +70,7 @@ test_create_user_already_exists() {
   username="$USERNAME_ONE"
   password="$PASSWORD_ONE"
 
-  run setup_user "$username" "123456" "admin"
-  assert_success "error setting up user"
-
+  setup_user "$username" "123456" "admin" || fail "error setting up user"
   if create_user "$username" "123456" "admin"; then
     fail "'user already exists' error not returned"
   fi

tests/util_list_objects.sh

@@ -166,35 +166,3 @@ list_objects_check_file_count() {
   fi
   return 0
 }
-
-check_object_listing_with_prefixes() {
-  if [ $# -ne 3 ]; then
-    log 2 "'check_object_listing_with_prefixes' requires bucket name, folder name, object name"
-    return 1
-  fi
-  if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "/"; then
-    log 2 "error listing objects with delimiter '/'"
-    return 1
-  fi
-  if ! prefix=$(echo "${objects[@]}" | jq -r ".CommonPrefixes[0].Prefix" 2>&1); then
-    log 2 "error getting object prefix from object list: $prefix"
-    return 1
-  fi
-  if [[ $prefix != "$2/" ]]; then
-    log 2 "prefix doesn't match (expected $2, actual $prefix/)"
-    return 1
-  fi
-  if ! list_objects_s3api_v1 "$BUCKET_ONE_NAME" "#"; then
-    log 2 "error listing objects with delimiter '#"
-    return 1
-  fi
-  if ! key=$(echo "${objects[@]}" | jq -r ".Contents[0].Key" 2>&1); then
-    log 2 "error getting key from object list: $key"
-    return 1
-  fi
-  if [[ $key != "$2/$3" ]]; then
-    log 2 "key doesn't match (expected $key, actual $2/$3)"
-    return 1
-  fi
-  return 0
-}

tests/util_policy.sh

@@ -205,20 +205,3 @@ get_and_check_policy() {
   fi
   return 0
 }
-
-put_and_check_for_malformed_policy() {
-  if [ $# -ne 2 ]; then
-    log 2 "'put_and_check_for_malformed_policy' requires bucket name, policy file"
-    return 1
-  fi
-  if put_bucket_policy "s3api" "$1" "$2"; then
-    log 2 "put succeeded despite malformed policy"
-    return 1
-  fi
-  # shellcheck disable=SC2154
-  if [[ "$put_bucket_policy_error" != *"MalformedPolicy"*"invalid action"* ]]; then
-    log 2 "invalid policy error: $put_bucket_policy_error"
-    return 1
-  fi
-  return 0
-}

tests/util_tags.sh

@@ -253,60 +253,3 @@ get_and_verify_object_tags() {
   fi
   return 0
 }
-
-verify_no_bucket_tags_rest() {
-  if [ $# -ne 1 ]; then
-    log 2 "'verify_no_bucket_tags_rest' requires bucket name"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/bucket_tagging.txt" ./tests/rest_scripts/get_bucket_tagging.sh); then
-    log 2 "error listing bucket tags: $result"
-    return 1
-  fi
-  if [ "$result" != "404" ]; then
-    log 2 "expected response code of '404', was '$result' (error: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt"))"
-    return 1
-  fi
-  return 0
-}
-
-add_verify_bucket_tags_rest() {
-  if [ $# -ne 3 ]; then
-    log 2 "'add_verify_bucket_tags_rest' requires bucket name, test key, test value"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" TAG_KEY="$2" TAG_VALUE="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" ./tests/rest_scripts/put_bucket_tagging.sh); then
-    log 2 "error putting bucket tags: $result"
-    return 1
-  fi
-  if [ "$result" != "204" ]; then
-    log 2 "expected response code of '204', was '$result' (error: $(cat "$TEST_FILE_FOLDER/result.txt"))"
-    return 1
-  fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$BUCKET_ONE_NAME" OUTPUT_FILE="$TEST_FILE_FOLDER/bucket_tagging.txt" ./tests/rest_scripts/get_bucket_tagging.sh); then
-    log 2 "error listing bucket tags: $result"
-    return 1
-  fi
-  if [ "$result" != "200" ]; then
-    log 2 "expected response code of '200', was '$result' (error: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt"))"
-    return 1
-  fi
-  log 5 "tags: $(cat "$TEST_FILE_FOLDER/bucket_tagging.txt")"
-  if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/bucket_tagging.txt" 2>&1); then
-    log 2 "error retrieving key: $key"
-    return 1
-  fi
-  if [ "$key" != "$2" ]; then
-    log 2 "key mismatch (expected '$2', actual '$key')"
-    return 1
-  fi
-  if ! value=$(xmllint --xpath '//*[local-name()="Value"]/text()' "$TEST_FILE_FOLDER/bucket_tagging.txt" 2>&1); then
-    log 2 "error retrieving value: $value"
-    return 1
-  fi
-  if [ "$value" != "$3" ]; then
-    log 2 "value mismatch (expected '$3', actual '$value')"
-    return 1
-  fi
-  return 0
-}

tests/util_users.sh

@@ -399,21 +399,4 @@ get_bucket_owner() {
   log 3 "bucket owner for bucket '$1' not found"
   bucket_owner=
   return 0
-}
-
-verify_user_cant_get_object() {
-  if [ $# -ne 6 ]; then
-    log 2 "'verify_user_cant_get_object' requires client, bucket, key, save file, username, password"
-    return 1
-  fi
-  if get_object_with_user "$1" "$2" "$3" "$4" "$5" "$6"; then
-    log 2 "get object with user succeeded despite lack of permissions"
-    return 1
-  fi
-  # shellcheck disable=SC2154
-  if [[ "$get_object_error" != *"Access Denied"* ]]; then
-    log 2 "invalid get object error: $get_object_error"
-    return 1
-  fi
-  return 0
 }