Merge pull request #653 from versity/test_cmdline_policy_more_tests

Test cmdline policy more tests

.github/workflows/system.yml

@@ -16,7 +16,9 @@ jobs:
             USERS_FOLDER: /tmp/iam1
             AWS_ENDPOINT_URL: https://127.0.0.1:7070
             RUN_SET: "s3cmd"
+            RECREATE_BUCKETS: "true"
             PORT: 7070
+            BACKEND: "posix"
           - set: 2
             LOCAL_FOLDER: /tmp/gw2
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
@@ -25,7 +27,9 @@ jobs:
             USERS_FOLDER: /tmp/iam2
             AWS_ENDPOINT_URL: https://127.0.0.1:7071
             RUN_SET: "s3"
+            RECREATE_BUCKETS: "true"
             PORT: 7071
+            BACKEND: "posix"
           - set: 3
             LOCAL_FOLDER: /tmp/gw3
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
@@ -34,7 +38,9 @@ jobs:
             USERS_FOLDER: /tmp/iam3
             AWS_ENDPOINT_URL: https://127.0.0.1:7072
             RUN_SET: "s3api"
+            RECREATE_BUCKETS: "true"
             PORT: 7072
+            BACKEND: "posix"
           - set: 4
             LOCAL_FOLDER: /tmp/gw4
             BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
@@ -43,17 +49,42 @@ jobs:
             USERS_FOLDER: /tmp/iam4
             AWS_ENDPOINT_URL: https://127.0.0.1:7073
             RUN_SET: "mc"
+            RECREATE_BUCKETS: "true"
             PORT: 7073
+            BACKEND: "posix"
           - set: 5
-            LOCAL_FOLDER: /tmp/gw4
-            BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
-            BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
+            LOCAL_FOLDER: /tmp/gw5
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
             IAM_TYPE: s3
             USERS_BUCKET: versity-gwtest-iam
             AWS_ENDPOINT_URL: https://127.0.0.1:7074
             RUN_SET: "aws-user"
+            RECREATE_BUCKETS: "true"
             PORT: 7074
-
+            BACKEND: "posix"
+          - set: 6
+            LOCAL_FOLDER: /tmp/gw6
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam6
+            AWS_ENDPOINT_URL: https://127.0.0.1:7075
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "false"
+            PORT: 7075
+            BACKEND: "posix"
+          - set: 7
+            LOCAL_FOLDER: /tmp/gw7
+            BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
+            BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
+            IAM_TYPE: folder
+            USERS_FOLDER: /tmp/iam7
+            AWS_ENDPOINT_URL: https://127.0.0.1:7076
+            RUN_SET: "aws"
+            RECREATE_BUCKETS: "true"
+            PORT: 7076
+            BACKEND: "s3"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -96,8 +127,8 @@ jobs:
           AWS_PROFILE: versity
           VERSITY_EXE: ${{ github.workspace }}/versitygw
           RUN_VERSITYGW: true
-          BACKEND: posix
-          RECREATE_BUCKETS: true
+          BACKEND: ${{ matrix.BACKEND }}
+          RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
           CERT: ${{ github.workspace }}/cert.pem
           KEY: ${{ github.workspace }}/versitygw.pem
           S3CMD_CONFIG: tests/s3cfg.local.default
@@ -109,6 +140,8 @@ jobs:
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
           export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
           export AWS_REGION=us-east-1
+          export AWS_ACCESS_KEY_ID_TWO=user
+          export AWS_SECRET_ACCESS_KEY_TWO=pass
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
           aws configure set aws_region $AWS_REGION --profile versity
@@ -117,22 +150,11 @@ jobs:
           openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
           openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
           mkdir $GOCOVERDIR $USERS_FOLDER
+          if [[ $RECREATE_BUCKETS == "false" ]]; then
+            BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
+          fi
           BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
 
-      #- name: Build and run, s3 backend
-      #  run: |
-      #    make testbin
-      #    export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export AWS_REGION=us-east-1
-      #    aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
-      #    aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
-      #    aws configure set aws_region $AWS_REGION --profile versity_s3
-      #    export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
-      #    export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
-      #    export WORKSPACE=$GITHUB_WORKSPACE
-      #    VERSITYGW_TEST_ENV=./tests/.env.s3 GOCOVERDIR=/tmp/cover ./tests/run_all.sh
-
       - name: Coverage report
         run: |
           go tool covdata percent -i=cover

auth/bucket_policy.go

@@ -16,12 +16,22 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 
 	"github.com/versity/versitygw/s3err"
 )
 
+var (
+	errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidResource = errors.New("Policy has invalid resource")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidPrincipal = errors.New("Invalid principal in policy")
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	errInvalidAction = errors.New("Policy has invalid action")
+)
+
 type BucketPolicy struct {
 	Statement []BucketPolicyItem `json:"Statement"`
 }
@@ -75,11 +85,14 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
 
 	for action := range bpi.Actions {
 		isObjectAction := action.IsObjectAction()
-		if isObjectAction && !containsObjectAction {
-			return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
+		if isObjectAction == nil {
+			break
 		}
-		if !isObjectAction && !containsBucketAction {
-			return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
+		if *isObjectAction && !containsObjectAction {
+			return errResourceMismatch
+		}
+		if !*isObjectAction && !containsBucketAction {
+			return errResourceMismatch
 		}
 	}
 
@@ -108,6 +121,11 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
 		return getMalformedPolicyError(err)
 	}
 
+	if len(policy.Statement) == 0 {
+		//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+		return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
+	}
+
 	if err := policy.Validate(bucket, iam); err != nil {
 		return getMalformedPolicyError(err)
 	}

auth/bucket_policy_actions.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -119,7 +118,7 @@ var supportedObjectActionList = map[Action]struct{}{
 // Validates Action: it should either wildcard match with supported actions list or be in it
 func (a Action) IsValid() error {
 	if !strings.HasPrefix(string(a), "s3:") {
-		return fmt.Errorf("invalid action: %v", a)
+		return errInvalidAction
 	}
 
 	if a == AllActions {
@@ -134,31 +133,39 @@ func (a Action) IsValid() error {
 			}
 		}
 
-		return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
+		return errInvalidAction
 	}
 
 	_, found := supportedActionList[a]
 	if !found {
-		return fmt.Errorf("unsupported action: %v", a)
+		return errInvalidAction
 	}
 	return nil
 }
 
+func getBoolPtr(bl bool) *bool {
+	return &bl
+}
+
 // Checks if the action is object action
-func (a Action) IsObjectAction() bool {
+// nil points to 's3:*'
+func (a Action) IsObjectAction() *bool {
+	if a == AllActions {
+		return nil
+	}
 	if a[len(a)-1] == '*' {
 		pattern := strings.TrimSuffix(string(a), "*")
 		for act := range supportedObjectActionList {
 			if strings.HasPrefix(string(act), pattern) {
-				return true
+				return getBoolPtr(true)
 			}
 		}
 
-		return false
+		return getBoolPtr(false)
 	}
 
 	_, found := supportedObjectActionList[a]
-	return found
+	return &found
 }
 
 func (a Action) WildCardMatch(act Action) bool {
@@ -177,7 +184,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("actions can't be empty")
+			return errInvalidAction
 		}
 		*a = make(Actions)
 		for _, s := range ss {
@@ -190,7 +197,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("actions can't be empty")
+				return errInvalidAction
 			}
 			*a = make(Actions)
 			err = a.Add(s)

auth/bucket_policy_effect.go

@@ -30,5 +30,6 @@ func (bpat BucketPolicyAccessType) Validate() error {
 		return nil
 	}
 
-	return fmt.Errorf("invalid effect: %v", bpat)
+	//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
+	return fmt.Errorf("Invalid effect: %v", bpat)
 }

auth/bucket_policy_principals.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 )
 
 type Principals map[string]struct{}
@@ -37,7 +36,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		for _, s := range ss {
@@ -46,7 +45,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &s); err == nil {
 		if s == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(s)
@@ -54,7 +53,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		return nil
 	} else if err = json.Unmarshal(data, &k); err == nil {
 		if k.AWS == "" {
-			return fmt.Errorf("principals can't be empty")
+			return errInvalidPrincipal
 		}
 		*p = make(Principals)
 		p.Add(k.AWS)
@@ -66,7 +65,7 @@ func (p *Principals) UnmarshalJSON(data []byte) error {
 		}
 		if err = json.Unmarshal(data, &sk); err == nil {
 			if len(sk.AWS) == 0 {
-				return fmt.Errorf("principals can't be empty")
+				return errInvalidPrincipal
 			}
 			*p = make(Principals)
 			for _, s := range sk.AWS {
@@ -98,7 +97,7 @@ func (p Principals) Validate(iam IAMService) error {
 		if len(p) == 1 {
 			return nil
 		}
-		return fmt.Errorf("principals should either contain * or user access keys")
+		return errInvalidPrincipal
 	}
 
 	accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -106,7 +105,7 @@ func (p Principals) Validate(iam IAMService) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user accounts don't exist: %v", accs)
+		return errInvalidPrincipal
 	}
 
 	return nil

auth/bucket_policy_resources.go

@@ -16,7 +16,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"strings"
 )
 
@@ -30,7 +29,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 	var err error
 	if err = json.Unmarshal(data, &ss); err == nil {
 		if len(ss) == 0 {
-			return fmt.Errorf("resources can't be empty")
+			return errInvalidResource
 		}
 		*r = make(Resources)
 		for _, s := range ss {
@@ -43,7 +42,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 		var s string
 		if err = json.Unmarshal(data, &s); err == nil {
 			if s == "" {
-				return fmt.Errorf("resources can't be empty")
+				return errInvalidResource
 			}
 			*r = make(Resources)
 			err = r.Add(s)
@@ -60,12 +59,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
 func (r Resources) Add(rc string) error {
 	ok, pattern := isValidResource(rc)
 	if !ok {
-		return fmt.Errorf("invalid resource: %v", rc)
-	}
-
-	_, found := r[pattern]
-	if found {
-		return fmt.Errorf("duplicate resource: %v", rc)
+		return errInvalidResource
 	}
 
 	r[pattern] = struct{}{}
@@ -99,7 +93,7 @@ func (r Resources) ContainsBucketPattern() bool {
 func (r Resources) Validate(bucket string) error {
 	for resource := range r {
 		if !strings.HasPrefix(resource, bucket) {
-			return fmt.Errorf("incorrect bucket name in %v", resource)
+			return errInvalidResource
 		}
 	}
 

auth/iam.go

@@ -37,12 +37,32 @@ type Account struct {
 	GroupID int    `json:"groupID"`
 }
 
+// Mutable props, which could be changed when updating an IAM account
+type MutableProps struct {
+	Secret  *string `json:"secret"`
+	UserID  *int    `json:"userID"`
+	GroupID *int    `json:"groupID"`
+}
+
+func updateAcc(acc *Account, props MutableProps) {
+	if props.Secret != nil {
+		acc.Secret = *props.Secret
+	}
+	if props.GroupID != nil {
+		acc.GroupID = *props.GroupID
+	}
+	if props.UserID != nil {
+		acc.UserID = *props.UserID
+	}
+}
+
 // IAMService is the interface for all IAM service implementations
 //
 //go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
 type IAMService interface {
 	CreateAccount(account Account) error
 	GetUserAccount(access string) (Account, error)
+	UpdateUserAccount(access string, props MutableProps) error
 	DeleteUserAccount(access string) error
 	ListUserAccounts() ([]Account, error)
 	Shutdown() error
@@ -65,6 +85,8 @@ type Opts struct {
 	LDAPAccessAtr          string
 	LDAPSecretAtr          string
 	LDAPRoleAtr            string
+	LDAPUserIdAtr          string
+	LDAPGroupIdAtr         string
 	VaultEndpointURL       string
 	VaultSecretStoragePath string
 	VaultMountPath         string
@@ -96,8 +118,8 @@ func New(o *Opts) (IAMService, error) {
 		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
 		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
-			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
-			o.LDAPObjClasses)
+			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
+			o.LDAPGroupIdAtr, o.LDAPObjClasses)
 		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
 	case o.S3Endpoint != "":
 		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,

auth/iam_cache.go

@@ -66,6 +66,21 @@ func (i *icache) get(k string) (Account, bool) {
 	return v.value, true
 }
 
+func (i *icache) update(k string, props MutableProps) {
+	i.Lock()
+	defer i.Unlock()
+
+	item, found := i.items[k]
+	if found {
+		updateAcc(&item.value, props)
+
+		// refresh the expiration date
+		item.exp = time.Now().Add(i.expire)
+
+		i.items[k] = item
+	}
+}
+
 func (i *icache) Delete(k string) {
 	i.Lock()
 	delete(i.items, k)
@@ -166,6 +181,16 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
 	return nil
 }
 
+func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
+	err := c.service.UpdateUserAccount(access, props)
+	if err != nil {
+		return err
+	}
+
+	c.iamcache.update(access, props)
+	return nil
+}
+
 // ListUserAccounts is a passthrough to the underlying service and
 // does not make use of the cache
 func (c *IAMCache) ListUserAccounts() ([]Account, error) {

auth/iam_internal.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 )
 
@@ -32,6 +33,13 @@ const (
 
 // IAMServiceInternal manages the internal IAM service
 type IAMServiceInternal struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
 	dir string
 }
 
@@ -62,6 +70,9 @@ func NewInternal(dir string) (*IAMServiceInternal, error) {
 // CreateAccount creates a new IAM account. Returns an error if the account
 // already exists.
 func (s *IAMServiceInternal) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -86,6 +97,9 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
 // GetUserAccount retrieves account info for the requested user. Returns
 // ErrNoSuchUser if the account does not exist.
 func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -99,9 +113,41 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+// UpdateUserAccount updates the specified user account fields. Returns
+// ErrNoSuchUser if the account does not exist.
+func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storeIAM(func(data []byte) ([]byte, error) {
+		conf, err := parseIAM(data)
+		if err != nil {
+			return nil, fmt.Errorf("get iam data: %w", err)
+		}
+
+		acc, found := conf.AccessAccounts[access]
+		if !found {
+			return nil, ErrNoSuchUser
+		}
+
+		updateAcc(&acc, props)
+		conf.AccessAccounts[access] = acc
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize iam: %w", err)
+		}
+
+		return b, nil
+	})
+}
+
 // DeleteUserAccount deletes the specified user account. Does not check if
 // account exists.
 func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	return s.storeIAM(func(data []byte) ([]byte, error) {
 		conf, err := parseIAM(data)
 		if err != nil {
@@ -121,6 +167,9 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
 
 // ListUserAccounts lists all the user accounts stored.
 func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getIAM()
 	if err != nil {
 		return []Account{}, fmt.Errorf("get iam data: %w", err)

auth/iam_ldap.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-ldap/ldap/v3"
@@ -28,12 +29,15 @@ type LdapIAMService struct {
 	accessAtr  string
 	secretAtr  string
 	roleAtr    string
+	groupIdAtr string
+	userIdAtr  string
 }
 
 var _ IAMService = &LdapIAMService{}
 
-func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
-	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
+func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
+	if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
+		secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
 		return nil, fmt.Errorf("required parameters list not fully provided")
 	}
 	conn, err := ldap.DialURL(url)
@@ -52,15 +56,19 @@ func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objCl
 		accessAtr:  accAtr,
 		secretAtr:  secAtr,
 		roleAtr:    roleAtr,
+		userIdAtr:  userIdAtr,
+		groupIdAtr: groupIdAtr,
 	}, nil
 }
 
 func (ld *LdapIAMService) CreateAccount(account Account) error {
-	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
+	userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
 	userEntry.Attribute("objectClass", ld.objClasses)
 	userEntry.Attribute(ld.accessAtr, []string{account.Access})
 	userEntry.Attribute(ld.secretAtr, []string{account.Secret})
 	userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
+	userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
+	userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
 
 	err := ld.conn.Add(userEntry)
 	if err != nil {
@@ -79,7 +87,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
 		nil,
 	)
 
@@ -88,14 +96,48 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
 		return Account{}, err
 	}
 
+	if len(result.Entries) == 0 {
+		return Account{}, ErrNoSuchUser
+	}
+
 	entry := result.Entries[0]
+	groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
+	}
+	userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
+	if err != nil {
+		return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
+	}
 	return Account{
-		Access: entry.GetAttributeValue(ld.accessAtr),
-		Secret: entry.GetAttributeValue(ld.secretAtr),
-		Role:   Role(entry.GetAttributeValue(ld.roleAtr)),
+		Access:  entry.GetAttributeValue(ld.accessAtr),
+		Secret:  entry.GetAttributeValue(ld.secretAtr),
+		Role:    Role(entry.GetAttributeValue(ld.roleAtr)),
+		GroupID: groupId,
+		UserID:  userId,
 	}, nil
 }
 
+func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
+	if props.Secret != nil {
+		req.Replace(ld.secretAtr, []string{*props.Secret})
+	}
+	if props.GroupID != nil {
+		req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
+	}
+	if props.UserID != nil {
+		req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
+	}
+
+	err := ld.conn.Modify(req)
+	//TODO: Handle non existing user case
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (ld *LdapIAMService) DeleteUserAccount(access string) error {
 	delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
 
@@ -120,7 +162,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 		0,
 		false,
 		fmt.Sprintf("(&%v)", searchFilter),
-		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
+		[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
 		nil,
 	)
 
@@ -131,10 +173,20 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
 
 	result := []Account{}
 	for _, el := range resp.Entries {
+		groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
+		}
+		userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
+		}
 		result = append(result, Account{
-			Access: el.GetAttributeValue(ld.accessAtr),
-			Secret: el.GetAttributeValue(ld.secretAtr),
-			Role:   Role(el.GetAttributeValue(ld.roleAtr)),
+			Access:  el.GetAttributeValue(ld.accessAtr),
+			Secret:  el.GetAttributeValue(ld.secretAtr),
+			Role:    Role(el.GetAttributeValue(ld.roleAtr)),
+			GroupID: groupId,
+			UserID:  userId,
 		})
 	}
 

auth/iam_s3_object.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -41,6 +42,14 @@ import (
 // coming from iAMConfig and iamFile in iam_internal.
 
 type IAMServiceS3 struct {
+	// This mutex will help with racing updates to the IAM data
+	// from multiple requests to this gateway instance, but
+	// will not help with racing updates to multiple load balanced
+	// gateway instances. This is a limitation of the internal
+	// IAM service. All account updates should be sent to a single
+	// gateway instance if possible.
+	sync.RWMutex
+
 	access        string
 	secret        string
 	region        string
@@ -97,6 +106,9 @@ func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug
 }
 
 func (s *IAMServiceS3) CreateAccount(account Account) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -112,6 +124,9 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
 }
 
 func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return Account{}, err
@@ -125,7 +140,30 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
 	return acct, nil
 }
 
+func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
+	s.Lock()
+	defer s.Unlock()
+
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	acc, ok := conf.AccessAccounts[access]
+	if !ok {
+		return ErrNoSuchUser
+	}
+
+	updateAcc(&acc, props)
+	conf.AccessAccounts[access] = acc
+
+	return s.storeAccts(conf)
+}
+
 func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	s.Lock()
+	defer s.Unlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return err
@@ -141,6 +179,9 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
 }
 
 func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	s.RLock()
+	defer s.RUnlock()
+
 	conf, err := s.getAccounts()
 	if err != nil {
 		return nil, err

auth/iam_single.go

@@ -35,6 +35,11 @@ func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
 	return Account{}, ErrNoSuchUser
 }
 
+// UpdateUserAccount no accounts in single tenant mode
+func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
+	return ErrNotSupported
+}
+
 // DeleteUserAccount no accounts in single tenant mode
 func (IAMServiceSingle) DeleteUserAccount(access string) error {
 	return ErrNotSupported

auth/iam_vault.go

@@ -140,6 +140,28 @@ func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
 	return acc, nil
 }
 
+func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
+	//TODO: We need something like a transaction here ?
+	acc, err := vt.GetUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	updateAcc(&acc, props)
+
+	err = vt.DeleteUserAccount(access)
+	if err != nil {
+		return err
+	}
+
+	err = vt.CreateAccount(acc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (vt *VaultIAMService) DeleteUserAccount(access string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)

backend/azure/azure.go

@@ -36,6 +36,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/versity/versitygw/auth"
@@ -176,7 +177,12 @@ func (az *Azure) CreateBucket(ctx context.Context, input *s3.CreateBucketInput,
 }
 
 func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	pager := az.client.NewListContainersPager(nil)
+	pager := az.client.NewListContainersPager(
+		&service.ListContainersOptions{
+			Include: service.ListContainersInclude{
+				Metadata: true,
+			},
+		})
 
 	var buckets []s3response.ListAllMyBucketsEntry
 	var result s3response.ListAllMyBucketsResult
@@ -187,11 +193,26 @@ func (az *Azure) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s
 			return result, azureErrToS3Err(err)
 		}
 		for _, v := range resp.ContainerItems {
-			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name: *v.Name,
-				// TODO: using modification date here instead of creation, is that ok?
-				CreationDate: *v.Properties.LastModified,
-			})
+			if isAdmin {
+				buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+					Name: *v.Name,
+					// TODO: using modification date here instead of creation, is that ok?
+					CreationDate: *v.Properties.LastModified,
+				})
+			} else {
+				acl, err := getAclFromMetadata(v.Metadata, keyAclLower)
+				if err != nil {
+					return result, err
+				}
+
+				if acl.Owner == owner {
+					buckets = append(buckets, s3response.ListAllMyBucketsEntry{
+						Name: *v.Name,
+						// TODO: using modification date here instead of creation, is that ok?
+						CreationDate: *v.Properties.LastModified,
+					})
+				}
+			}
 		}
 	}
 

backend/common.go

@@ -18,6 +18,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
@@ -98,6 +99,14 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
 	return startOffset, endOffset - startOffset + 1, nil
 }
 
+func CreateExceedingRangeErr(objSize int64) s3err.APIError {
+	return s3err.APIError{
+		Code:           "InvalidArgument",
+		Description:    fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	}
+}
+
 func GetMultipartMD5(parts []types.CompletedPart) string {
 	var partsEtagBytes []byte
 	for _, part := range parts {

backend/posix/posix.go

@@ -656,7 +656,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM
 	}
 
 	// cleanup tmp dirs
-	os.RemoveAll(upiddir)
+	os.RemoveAll(filepath.Join(bucket, objdir, uploadID))
 	// use Remove for objdir in case there are still other uploads
 	// for same object name outstanding, this will fail if there are
 	os.Remove(filepath.Join(bucket, objdir))
@@ -1209,7 +1209,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	}
 
 	if startOffset+length > fi.Size()+1 {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidRange)
+		return s3response.CopyObjectResult{}, backend.CreateExceedingRangeErr(fi.Size())
 	}
 
 	f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),

backend/scoutfs/scoutfs.go

@@ -969,15 +969,9 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 }
 
 func isNoAttr(err error) bool {
-	if err == nil {
-		return false
-	}
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {
 		return true
 	}
-	if err == errNoData {
-		return true
-	}
 	return false
 }

backend/scoutfs/with_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build !freebsd && !openbsd && !netbsd
-// +build !freebsd,!openbsd,!netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENODATA
-)

backend/scoutfs/without_enodata.go

@@ -1,24 +0,0 @@
-// Copyright 2024 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//go:build freebsd || openbsd || netbsd
-// +build freebsd openbsd netbsd
-
-package scoutfs
-
-import "syscall"
-
-var (
-	errNoData = syscall.ENOATTR
-)

cmd/versitygw/admin.go

@@ -82,6 +82,34 @@ func adminCommand() *cli.Command {
 					},
 				},
 			},
+			{
+				Name:   "update-user",
+				Usage:  "Updates a user account",
+				Action: updateUser,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "access",
+						Usage:    "user access key id to be updated",
+						Required: true,
+						Aliases:  []string{"a"},
+					},
+					&cli.StringFlag{
+						Name:    "secret",
+						Usage:   "secret access key for the new user",
+						Aliases: []string{"s"},
+					},
+					&cli.IntFlag{
+						Name:    "user-id",
+						Usage:   "userID for the new user",
+						Aliases: []string{"ui"},
+					},
+					&cli.IntFlag{
+						Name:    "group-id",
+						Usage:   "groupID for the new user",
+						Aliases: []string{"gi"},
+					},
+				},
+			},
 			{
 				Name:   "delete-user",
 				Usage:  "Delete a user",
@@ -276,6 +304,63 @@ func deleteUser(ctx *cli.Context) error {
 	return nil
 }
 
+func updateUser(ctx *cli.Context) error {
+	access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
+	props := auth.MutableProps{}
+	if ctx.IsSet("secret") {
+		props.Secret = &secret
+	}
+	if ctx.IsSet("user-id") {
+		props.UserID = &userId
+	}
+	if ctx.IsSet("group-id") {
+		props.GroupID = &groupId
+	}
+
+	propsJSON, err := json.Marshal(props)
+	if err != nil {
+		return fmt.Errorf("failed to parse user attributes: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256(propsJSON)
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := initHTTPClient()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("%s", body)
+	}
+
+	fmt.Printf("%s\n", body)
+
+	return nil
+}
+
 func listUsers(ctx *cli.Context) error {
 	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
 	if err != nil {

cmd/versitygw/main.go

@@ -56,6 +56,7 @@ var (
 	ldapURL, ldapBindDN, ldapPassword        string
 	ldapQueryBase, ldapObjClasses            string
 	ldapAccessAtr, ldapSecAtr, ldapRoleAtr   string
+	ldapUserIdAtr, ldapGroupIdAtr            string
 	vaultEndpointURL, vaultSecretStoragePath string
 	vaultMountPath, vaultRootToken           string
 	vaultRoleId, vaultRoleSecret             string
@@ -331,6 +332,18 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IAM_LDAP_ROLE_ATR"},
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-user-id-atr",
+			Usage:       "ldap server user id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_USER_ID_ATR"},
+			Destination: &ldapUserIdAtr,
+		},
+		&cli.StringFlag{
+			Name:        "iam-ldap-group-id-atr",
+			Usage:       "ldap server user group id attribute name",
+			EnvVars:     []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
+			Destination: &ldapGroupIdAtr,
+		},
 		&cli.StringFlag{
 			Name:        "iam-vault-endpoint-url",
 			Usage:       "vault server url",
@@ -569,6 +582,8 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 		LDAPAccessAtr:          ldapAccessAtr,
 		LDAPSecretAtr:          ldapSecAtr,
 		LDAPRoleAtr:            ldapRoleAtr,
+		LDAPUserIdAtr:          ldapUserIdAtr,
+		LDAPGroupIdAtr:         ldapGroupIdAtr,
 		VaultEndpointURL:       vaultEndpointURL,
 		VaultSecretStoragePath: vaultSecretStoragePath,
 		VaultMountPath:         vaultMountPath,

go.mod

@@ -4,23 +4,23 @@ go 1.21.0
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/aws/aws-sdk-go-v2 v1.27.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1
+	github.com/aws/aws-sdk-go-v2 v1.30.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1
 	github.com/aws/smithy-go v1.20.2
 	github.com/go-ldap/ldap/v3 v3.4.8
 	github.com/gofiber/fiber/v2 v2.52.4
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault-client-go v0.4.3
-	github.com/nats-io/nats.go v1.35.0
+	github.com/nats-io/nats.go v1.36.0
 	github.com/pkg/xattr v0.4.9
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.3
 	github.com/urfave/cli/v2 v2.27.2
-	github.com/valyala/fasthttp v1.54.0
+	github.com/valyala/fasthttp v1.55.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
 	golang.org/x/sys v0.21.0
 )
@@ -30,11 +30,11 @@ require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.21.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -58,18 +58,18 @@ require (
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.18
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.18
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.21
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.21
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect

go.sum

@@ -1,7 +1,7 @@
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
@@ -21,42 +21,42 @@ github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7V
 github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go-v2 v1.27.2 h1:pLsTXqX93rimAOZG2FIYraDQstZaaGVVN4tNw65v0h8=
-github.com/aws/aws-sdk-go-v2 v1.27.2/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2 v1.30.0 h1:6qAwtzlfcTtcL8NHtbDQAqgM5s6NDipQTkPxyH/6kAA=
+github.com/aws/aws-sdk-go-v2 v1.30.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
-github.com/aws/aws-sdk-go-v2/config v1.27.18 h1:wFvAnwOKKe7QAyIxziwSKjmer9JBMH1vzIL6W+fYuKk=
-github.com/aws/aws-sdk-go-v2/config v1.27.18/go.mod h1:0xz6cgdX55+kmppvPm2IaKzIXOheGJhAufacPJaXZ7c=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.18 h1:D/ALDWqK4JdY3OFgA2thcPO1c9aYTT5STS/CvnkqY1c=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.18/go.mod h1:JuitCWq+F5QGUrmMPsk945rop6bB57jdscu+Glozdnc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 h1:dDgptDO9dxeFkXy+tEgVkzSClHZje/6JkPW5aZyEvrQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5/go.mod h1:gjvE2KBUgUQhcv89jqxrIxH9GaKs1JbZzWejj/DaHGA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24 h1:FzNwpVTZDCvm597Ty6mGYvxTolyC1oup0waaKntZI4E=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24/go.mod h1:wM9NElT/Wn6n3CT1eyVcXtfCy8lSVjjQXfdawQbSShc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 h1:cy8ahBJuhtM8GTTSyOkfy6WVPV1IE+SS5/wfXUYuulw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9/go.mod h1:CZBXGLaJnEZI6EVNcPd7a6B5IC5cA/GkRWtu9fp3S6Y=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 h1:A4SYk07ef04+vxZToz9LWvAXl9LW0NClpPpMsi31cz0=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9/go.mod h1:5jJcHuwDagxN+ErjQ3PU3ocf6Ylc/p9x+BLO/+X4iXw=
+github.com/aws/aws-sdk-go-v2/config v1.27.21 h1:yPX3pjGCe2hJsetlmGNB4Mngu7UPmvWPzzWCv1+boeM=
+github.com/aws/aws-sdk-go-v2/config v1.27.21/go.mod h1:4XtlEU6DzNai8RMbjSF5MgGZtYvrhBP/aKZcRtZAVdM=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.21 h1:pjAqgzfgFhTv5grc7xPHtXCAaMapzmwA7aU+c/SZQGw=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.21/go.mod h1:nhK6PtBlfHTUDVmBLr1dg+WHCOCK+1Fu/WQyVHPsgNQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8 h1:FR+oWPFb/8qMVYMWN98bUZAGqPvLHiyqg1wqQGfUAXY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.8/go.mod h1:EgSKcHiuuakEIxJcKGzVNWh5srVAQ3jKaSrBGRYvM48=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1 h1:D9VqWMuw7lJAX6d5eINfRQ/PkvtcJAK3Qmd6f6xEeUw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.1/go.mod h1:ckvBx7codI4wzc5inOfDp5ZbK7TjMFa7eXwmLvXQrRk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12 h1:SJ04WXGTwnHlWIODtC5kJzKbeuHt+OUNOgKg7nfnUGw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.12/go.mod h1:FkpvXhA92gb3GE9LD6Og0pHHycTxW7xGpnEh5E7Opwo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12 h1:hb5KgeYfObi5MHkSSZMEudnIvX30iB+E21evI4r6BnQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.12/go.mod h1:CroKe/eWJdyfy9Vx4rljP5wTUjNJfb+fPz1uMYUhEGM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9 h1:vHyZxoLVOgrI8GqX7OMHLXp4YYoxeEsrjweXKpye+ds=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9/go.mod h1:z9VXZsWA2BvZNH1dT0ToUYwMu/CR9Skkj/TBX+mceZw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 h1:DXFWyt7ymx/l1ygdyTTS0X923e+Q2wXIxConJzrgwc0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12/go.mod h1:mVOr/LbvaNySK1/BTy4cBOCjhCNY2raWBwK4v+WR5J4=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11 h1:4vt9Sspk59EZyHCAEMaktHKiq0C09noRTQorXD/qV+s=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11/go.mod h1:5jHR79Tv+Ccq6rwYh+W7Nptmw++WiFafMfR42XhwNl8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 h1:o4T+fKxA3gTMcluBNZZXE9DNaMkJuUL1O3mffCUjoJo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11/go.mod h1:84oZdJ+VjuJKs9v1UTC9NaodRZRseOXCTgku+vQJWR8=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9 h1:TE2i0A9ErH1YfRSvXfCr2SQwfnqsoJT9nPQ9kj0lkxM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9/go.mod h1:9TzXX3MehQNGPwCZ3ka4CpwQsoAMWSF48/b+De9rfVM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1 h1:UAxBuh0/8sFJk1qOkvOKewP5sWeWaTPDknbQz0ZkDm0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1/go.mod h1:hWjsYGjVuqCgfoveVcVFPXIWgz0aByzwaxKlN1StKcM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 h1:gEYM2GSpr4YNWc6hCd5nod4+d4kd9vWIAWrmGuLdlMw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.20.11/go.mod h1:gVvwPdPNYehHSP9Rs7q27U1EU+3Or2ZpXvzAYJNh63w=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 h1:iXjh3uaH3vsVcnyZX7MqCoCfcyxIrVE9iOQruRaWPrQ=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5/go.mod h1:5ZXesEuy/QcO0WUnt+4sDkxhdXRHTu2yG0uCSH8B6os=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 h1:M/1u4HBpwLuMtjlxuI2y6HoVLzF5e2mfxHCg7ZVMYmk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.28.12/go.mod h1:kcfd+eTdEi/40FIbLq4Hif3XMXnl5b/+t/KTfLt9xIk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 h1:oWccitSnByVU74rQRHac4gLfDqjB6Z1YQGOY/dXKedI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14/go.mod h1:8SaZBlQdCLrc/2U3CEO48rYj9uR8qRsPRkmzwNM52pM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 h1:zSDPny/pVnkqABXYRicYuPf9z2bTqfH13HT3v6UheIk=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14/go.mod h1:3TTcI5JSzda1nw/pkVC9dhgLre0SNBFj2lYS4GctXKI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 h1:tzha+v1SCEBpXWEuw6B/+jm4h5z8hZbTpXz0zRZqTnw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12/go.mod h1:n+nt2qjHGoseWeLHt1vEr6ZRCCxIN2KcNpJxBcYQSwI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 h1:wsg9Z/vNnCmxWikfGIoOlnExtEU459cR+2d+iDJ8elo=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1/go.mod h1:8rDw3mVwmvIWWX/+LWY3PPIMZuwnQdJMCt0iVFVT3qw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.21.1 h1:sd0BsnAvLH8gsp2e3cbaIr+9D7T1xugueQ7V/zUAsS4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.21.1/go.mod h1:lcQG/MmxydijbeTOp04hIuJwXGWPZGI3bwdFDGRTv14=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1 h1:1uEFNNskK/I1KoZ9Q8wJxMz5V9jyBlsiaNrM7vA3YUQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.25.1/go.mod h1:z0P8K+cBIsFXUr5rzo/psUeJ20XjPN0+Nn8067Nd+E4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.29.1 h1:myX5CxqXE0QMZNja6FA1/FSE3Vu1rVmeUmpJMMzeZg0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.29.1/go.mod h1:N2mQiucsO0VwK9CYuS4/c2n6Smeh1v47Rz3dWCPFLdE=
 github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
 github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
@@ -114,8 +114,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -127,8 +127,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk=
-github.com/nats-io/nats.go v1.35.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU=
+github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -170,8 +170,8 @@ github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
 github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
-github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=

s3api/admin-router.go

@@ -32,6 +32,9 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
 	// DeleteUsers admin api
 	app.Patch("/delete-user", controller.DeleteUser)
 
+	// UpdateUser admin api
+	app.Patch("/update-user", controller.UpdateUser)
+
 	// ListUsers admin api
 	app.Patch("/list-users", controller.ListUsers)
 

s3api/controllers/admin.go

@@ -17,6 +17,7 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
@@ -35,31 +36,69 @@ func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController
 func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	var usr auth.Account
 	err := json.Unmarshal(ctx.Body(), &usr)
 	if err != nil {
-		return fmt.Errorf("failed to parse request body: %w", err)
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("failed to parse request body: %w", err).Error())
 	}
 
 	if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
-		return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
+		return ctx.Status(fiber.StatusBadRequest).SendString("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
 	}
 
 	err = c.iam.CreateAccount(usr)
 	if err != nil {
-		return fmt.Errorf("failed to create user: %w", err)
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to create user: %w", err).Error()
+
+		if strings.Contains(msg, "user already exists") {
+			status = fiber.StatusConflict
+		}
+
+		return ctx.Status(status).SendString(msg)
 	}
 
-	return ctx.SendString("The user has been created successfully")
+	return ctx.Status(fiber.StatusCreated).SendString("The user has been created successfully")
+}
+
+func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
+	acct := ctx.Locals("account").(auth.Account)
+	if acct.Role != "admin" {
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
+	}
+
+	access := ctx.Query("access")
+	if access == "" {
+		return ctx.Status(fiber.StatusBadRequest).SendString("missing user access parameter")
+	}
+
+	var props auth.MutableProps
+	if err := json.Unmarshal(ctx.Body(), &props); err != nil {
+		return ctx.Status(fiber.StatusBadRequest).SendString(fmt.Errorf("invalid request body %w", err).Error())
+	}
+
+	err := c.iam.UpdateUserAccount(access, props)
+	if err != nil {
+		status := fiber.StatusInternalServerError
+		msg := fmt.Errorf("failed to update user account: %w", err).Error()
+
+		if strings.Contains(msg, "user not found") {
+			status = fiber.StatusNotFound
+		}
+
+		return ctx.Status(status).SendString(msg)
+	}
+
+	return ctx.SendString("the user has been updated successfully")
 }
 
 func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 	access := ctx.Query("access")
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	err := c.iam.DeleteUserAccount(access)
@@ -73,7 +112,7 @@ func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
 func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	accs, err := c.iam.ListUserAccounts()
 	if err != nil {
@@ -86,7 +125,7 @@ func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
 func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 	owner := ctx.Query("owner")
 	bucket := ctx.Query("bucket")
@@ -96,7 +135,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 	if len(accs) > 0 {
-		return fmt.Errorf("user specified as the new bucket owner does not exist")
+		return ctx.Status(fiber.StatusNotFound).SendString("user specified as the new bucket owner does not exist")
 	}
 
 	err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
@@ -104,13 +143,13 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
 		return err
 	}
 
-	return ctx.Status(201).SendString("Bucket owner has been updated successfully")
+	return ctx.SendString("Bucket owner has been updated successfully")
 }
 
 func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
 	acct := ctx.Locals("account").(auth.Account)
 	if acct.Role != "admin" {
-		return fmt.Errorf("access denied: only admin users have access to this resource")
+		return ctx.Status(fiber.StatusForbidden).SendString("access denied: only admin users have access to this resource")
 	}
 
 	buckets, err := c.be.ListBucketsAndOwners(ctx.Context())

s3api/controllers/admin_test.go

@@ -85,7 +85,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
 			},
 			wantErr:    false,
-			statusCode: 200,
+			statusCode: 201,
 		},
 		{
 			name: "Admin-create-user-invalid-user-role",
@@ -94,7 +94,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 400,
 		},
 		{
 			name: "Admin-create-user-invalid-requester-role",
@@ -103,7 +103,7 @@ func TestAdminController_CreateUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -119,6 +119,122 @@ func TestAdminController_CreateUser(t *testing.T) {
 	}
 }
 
+func TestAdminController_UpdateUser(t *testing.T) {
+	type args struct {
+		req *http.Request
+	}
+
+	adminController := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return nil
+			},
+		},
+	}
+
+	app := fiber.New()
+
+	app.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	app.Patch("/update-user", adminController.UpdateUser)
+
+	appErr := fiber.New()
+
+	appErr.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
+		return ctx.Next()
+	})
+
+	appErr.Patch("/update-user", adminController.UpdateUser)
+
+	successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
+
+	adminControllerErr := AdminController{
+		iam: &IAMServiceMock{
+			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+				return auth.ErrNoSuchUser
+			},
+		},
+	}
+
+	appNotFound := fiber.New()
+
+	appNotFound.Use(func(ctx *fiber.Ctx) error {
+		ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
+		return ctx.Next()
+	})
+
+	appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
+
+	tests := []struct {
+		name       string
+		app        *fiber.App
+		args       args
+		wantErr    bool
+		statusCode int
+	}{
+		{
+			name: "Admin-update-user-success",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 200,
+		},
+		{
+			name: "Admin-update-user-missing-access",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-request-body",
+			app:  app,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 400,
+		},
+		{
+			name: "Admin-update-user-invalid-requester-role",
+			app:  appErr,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
+			},
+			wantErr:    false,
+			statusCode: 403,
+		},
+		{
+			name: "Admin-update-user-not-found",
+			app:  appNotFound,
+			args: args{
+				req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
+			},
+			wantErr:    false,
+			statusCode: 404,
+		},
+	}
+	for _, tt := range tests {
+		resp, err := tt.app.Test(tt.args.req)
+
+		if (err != nil) != tt.wantErr {
+			t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
+		}
+
+		if resp.StatusCode != tt.statusCode {
+			t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
+		}
+	}
+}
+
 func TestAdminController_DeleteUser(t *testing.T) {
 	type args struct {
 		req *http.Request
@@ -173,7 +289,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 	}
 	for _, tt := range tests {
@@ -251,7 +367,7 @@ func TestAdminController_ListUsers(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Admin-list-users-iam-error",
@@ -368,7 +484,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "Change-bucket-owner-check-account-server-error",
@@ -386,7 +502,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 404,
 		},
 		{
 			name: "Change-bucket-owner-success",
@@ -395,7 +511,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
 			},
 			wantErr:    false,
-			statusCode: 201,
+			statusCode: 200,
 		},
 	}
 	for _, tt := range tests {
@@ -455,7 +571,7 @@ func TestAdminController_ListBuckets(t *testing.T) {
 				req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
 			},
 			wantErr:    false,
-			statusCode: 500,
+			statusCode: 403,
 		},
 		{
 			name: "List-buckets-success",

s3api/controllers/iam_moq_test.go

@@ -33,6 +33,9 @@ var _ auth.IAMService = &IAMServiceMock{}
 //			ShutdownFunc: func() error {
 //				panic("mock out the Shutdown method")
 //			},
+//			UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
+//				panic("mock out the UpdateUserAccount method")
+//			},
 //		}
 //
 //		// use mockedIAMService in code that requires auth.IAMService
@@ -55,6 +58,9 @@ type IAMServiceMock struct {
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func() error
 
+	// UpdateUserAccountFunc mocks the UpdateUserAccount method.
+	UpdateUserAccountFunc func(access string, props auth.MutableProps) error
+
 	// calls tracks calls to the methods.
 	calls struct {
 		// CreateAccount holds details about calls to the CreateAccount method.
@@ -78,12 +84,20 @@ type IAMServiceMock struct {
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
 		}
+		// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
+		UpdateUserAccount []struct {
+			// Access is the access argument value.
+			Access string
+			// Props is the props argument value.
+			Props auth.MutableProps
+		}
 	}
 	lockCreateAccount     sync.RWMutex
 	lockDeleteUserAccount sync.RWMutex
 	lockGetUserAccount    sync.RWMutex
 	lockListUserAccounts  sync.RWMutex
 	lockShutdown          sync.RWMutex
+	lockUpdateUserAccount sync.RWMutex
 }
 
 // CreateAccount calls CreateAccountFunc.
@@ -235,3 +249,39 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
 	mock.lockShutdown.RUnlock()
 	return calls
 }
+
+// UpdateUserAccount calls UpdateUserAccountFunc.
+func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
+	if mock.UpdateUserAccountFunc == nil {
+		panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
+	}
+	callInfo := struct {
+		Access string
+		Props  auth.MutableProps
+	}{
+		Access: access,
+		Props:  props,
+	}
+	mock.lockUpdateUserAccount.Lock()
+	mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
+	mock.lockUpdateUserAccount.Unlock()
+	return mock.UpdateUserAccountFunc(access, props)
+}
+
+// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
+// Check the length with:
+//
+//	len(mockedIAMService.UpdateUserAccountCalls())
+func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
+	Access string
+	Props  auth.MutableProps
+} {
+	var calls []struct {
+		Access string
+		Props  auth.MutableProps
+	}
+	mock.lockUpdateUserAccount.RLock()
+	calls = mock.calls.UpdateUserAccount
+	mock.lockUpdateUserAccount.RUnlock()
+	return calls
+}

s3api/router.go

@@ -40,6 +40,9 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
 		// DeleteUsers admin api
 		app.Patch("/delete-user", adminController.DeleteUser)
 
+		// UpdateUser admin api
+		app.Patch("update-user", adminController.UpdateUser)
+
 		// ListUsers admin api
 		app.Patch("/list-users", adminController.ListUsers)
 

tests/README.md

@@ -10,6 +10,7 @@
    * **s3cmd**:  Instructions are [here](https://github.com/s3tools/s3cmd/blob/master/INSTALL.md).
    * **mc**:  Instructions are [here](https://min.io/docs/minio/linux/reference/minio-mc.html).
 3. Install BATS.  Instructions are [here](https://bats-core.readthedocs.io/en/stable/installation.html).
+4. If running on Mac OS, install **jq** with the command `brew install jq`.
 4. Create a `.secrets` file in the `tests` folder, and add the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` values to the file.
 5. Create a local AWS profile for connection to S3, and add the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` values for your account to the profile.  Example:
 ```
@@ -30,6 +31,10 @@
 8. Set `BUCKET_ONE_NAME` and `BUCKET_TWO_NAME` to the desired names of your buckets.  If you don't want them to be created each time, set `RECREATE_BUCKETS` to `false`.
 9. In the root repo folder, run single test group with `VERSITYGW_TEST_ENV=<env file> tests/run.sh <options>`.  To print options, run `tests/run.sh -h`.  To run all tests, run `VERSITYGW_TEST_ENV=<env file> tests/run_all.sh`.
 
+### Static Bucket Mode
+
+To preserve buckets while running tests, set `RECREATE_BUCKETS` to `false`.  Two utility functions are included, if needed, to create, and delete buckets for this:  `tests/setup_static.sh` and `tests/remove_static.sh`.
+
 ### S3 Backend
 
 Instructions are mostly the same; however, testing with the S3 backend requires two S3 accounts.  Ideally, these are two real accounts, but one can also be a dummy account that versity uses internally.

tests/commands/delete_bucket_policy.sh

@@ -5,14 +5,14 @@ delete_bucket_policy() {
     log 2 "delete bucket policy command requires command type, bucket"
     return 1
   fi
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     error=$(aws --no-verify-ssl s3api delete-bucket-policy --bucket "$2" 2>&1) || delete_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate delpolicy "s3://$2" 2>&1) || delete_result=$?
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure anonymous set none "$MC_ALIAS/$2" 2>&1) || delete_result=$?
   else
-    log 2 "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'delete bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $delete_result -ne 0 ]]; then

tests/commands/get_bucket_policy.sh

@@ -2,22 +2,22 @@
 
 get_bucket_policy() {
   if [[ $# -ne 2 ]]; then
-    echo "get bucket policy command requires command type, bucket"
+    log 2 "get bucket policy command requires command type, bucket"
     return 1
   fi
   local get_bucket_policy_result=0
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     get_bucket_policy_aws "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
     get_bucket_policy_s3cmd "$2" || get_bucket_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
     get_bucket_policy_mc "$2" || get_bucket_policy_result=$?
   else
-    echo "command 'get bucket policy' not implemented for '$1'"
+    log 2 "command 'get bucket policy' not implemented for '$1'"
     return 1
   fi
   if [[ $get_bucket_policy_result -ne 0 ]]; then
-    echo "error getting policy: $bucket_policy"
+    log 2 "error getting policy: $bucket_policy"
     return 1
   fi
   export bucket_policy
@@ -26,17 +26,17 @@ get_bucket_policy() {
 
 get_bucket_policy_aws() {
   if [[ $# -ne 1 ]]; then
-    echo "aws 'get bucket policy' command requires bucket"
+    log 2 "aws 'get bucket policy' command requires bucket"
     return 1
   fi
-  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || get_result=$?
+  policy_json=$(aws --no-verify-ssl s3api get-bucket-policy --bucket "$1" 2>&1) || local get_result=$?
   policy_json=$(echo "$policy_json" | grep -v "InsecureRequestWarning")
   log 5 "$policy_json"
   if [[ $get_result -ne 0 ]]; then
     if [[ "$policy_json" == *"(NoSuchBucketPolicy)"* ]]; then
       bucket_policy=
     else
-      echo "error getting policy: $policy_json"
+      log 2 "error getting policy: $policy_json"
       return 1
     fi
   else

tests/commands/get_object.sh

@@ -39,3 +39,24 @@ get_object_with_range() {
   fi
   return 0
 }
+
+get_object_with_user() {
+  if [ $# -ne 6 ]; then
+    log 2 "'get object with user' command requires command type, bucket, key, save location, aws ID, aws secret key"
+    return 1
+  fi
+  local exit_code=0
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    get_object_error=$(AWS_ACCESS_KEY_ID="$5" AWS_SECRET_ACCESS_KEY="$6" aws --no-verify-ssl s3api get-object --bucket "$2" --key "$3" "$4" 2>&1) || exit_code=$?
+  else
+    log 2 "'get object with user' command not implemented for '$1'"
+    return 1
+  fi
+  log 5 "put object exit code: $exit_code"
+  if [ $exit_code -ne 0 ]; then
+    log 2 "error getting object: $get_object_error"
+    export get_object_error
+    return 1
+  fi
+  return 0
+}

tests/commands/put_bucket_acl.sh

@@ -2,7 +2,7 @@
 
 put_bucket_acl() {
   if [[ $# -ne 3 ]]; then
-    log 2 "put bucket acl command requires command type, bucket name, acls"
+    log 2 "put bucket acl command requires command type, bucket name, acls or username"
     return 1
   fi
   local error=""
@@ -11,7 +11,7 @@ put_bucket_acl() {
     log 5 "bucket name: $2, acls: $3"
     error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$2" --access-control-policy "file://$3" 2>&1) || put_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$2" --acl-grant=read:ABCDEFG 2>&1) || put_result=$?
+    error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setacl "s3://$2" --acl-grant=read:"$3" 2>&1) || put_result=$?
   else
     log 2 "put_bucket_acl not implemented for '$1'"
     return 1
@@ -21,4 +21,16 @@ put_bucket_acl() {
     return 1
   fi
   return 0
+}
+
+put_bucket_canned_acl() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "'put bucket canned acl' command requires bucket name, canned ACL"
+    return 1
+  fi
+  if ! error=$(aws --no-verify-ssl s3api put-bucket-acl --bucket "$1" --acl "$2"); then
+    log 2 "error resetting bucket acls: $error"
+    return 1
+  fi
+  return 0
 }

tests/commands/put_bucket_policy.sh

@@ -5,18 +5,21 @@ put_bucket_policy() {
     log 2 "get bucket policy command requires command type, bucket, policy file"
     return 1
   fi
+  local put_policy_result=0
   if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
-    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_result=$?
+    policy=$(aws --no-verify-ssl s3api put-bucket-policy --bucket "$2" --policy "file://$3" 2>&1) || put_policy_result=$?
   elif [[ $1 == 's3cmd' ]]; then
-    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_result=$?
+    policy=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate setpolicy "$3" "s3://$2" 2>&1) || put_policy_result=$?
   elif [[ $1 == 'mc' ]]; then
-    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_result=$?
+    policy=$(mc --insecure anonymous set-json "$3" "$MC_ALIAS/$2" 2>&1) || put_policy_result=$?
   else
     log 2 "command 'put bucket policy' not implemented for '$1'"
     return 1
   fi
-  if [[ $put_result -ne 0 ]]; then
-    log 2 "error putting policy: $policy"
+  if [[ $put_policy_result -ne 0 ]]; then
+    put_bucket_policy_error=$policy
+    log 2 "error putting policy: $put_bucket_policy_error"
+    export put_bucket_policy_error
     return 1
   fi
   return 0

tests/env.sh

@@ -29,15 +29,17 @@ check_env_vars() {
 }
 
 check_universal_vars() {
-  if [ -z "$VERSITYGW_TEST_ENV" ]; then
-    if [ -r tests/.env ]; then
-      source tests/.env
+  if [[ $BYPASS_ENV_FILE != "true" ]]; then
+    if [ -z "$VERSITYGW_TEST_ENV" ]; then
+      if [ -r tests/.env ]; then
+        source tests/.env
+      else
+        log 3 "Warning: no .env file found in tests folder"
+      fi
     else
-      log 3 "Warning: no .env file found in tests folder"
+      # shellcheck source=./tests/.env.default
+      source "$VERSITYGW_TEST_ENV"
     fi
-  elif [[ $BYPASS_ENV_FILE != "true" ]]; then
-    # shellcheck source=./tests/.env.default
-    source "$VERSITYGW_TEST_ENV"
   fi
   if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
     # shellcheck source=./tests/.secrets

tests/integration/group-tests.go

@@ -286,6 +286,7 @@ func TestGetBucketAcl(s *S3Conf) {
 
 func TestPutBucketPolicy(s *S3Conf) {
 	PutBucketPolicy_non_existing_bucket(s)
+	PutBucketPolicy_empty_statement(s)
 	PutBucketPolicy_invalid_effect(s)
 	PutBucketPolicy_empty_actions_string(s)
 	PutBucketPolicy_empty_actions_array(s)
@@ -638,6 +639,7 @@ func GetIntTests() IntTests {
 		"GetBucketAcl_access_denied":                                          GetBucketAcl_access_denied,
 		"GetBucketAcl_success":                                                GetBucketAcl_success,
 		"PutBucketPolicy_non_existing_bucket":                                 PutBucketPolicy_non_existing_bucket,
+		"PutBucketPolicy_empty_statement":                                     PutBucketPolicy_empty_statement,
 		"PutBucketPolicy_invalid_effect":                                      PutBucketPolicy_invalid_effect,
 		"PutBucketPolicy_empty_actions_string":                                PutBucketPolicy_empty_actions_string,
 		"PutBucketPolicy_empty_actions_array":                                 PutBucketPolicy_empty_actions_array,

tests/integration/tests.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
@@ -5149,7 +5150,7 @@ func UploadPartCopy_greater_range_than_obj_size(s *S3Conf) error {
 			PartNumber:      &partNumber,
 		})
 		cancel()
-		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil {
+		if err := checkApiErr(err, backend.CreateExceedingRangeErr(int64(srcObjSize))); err != nil {
 			return err
 		}
 
@@ -6319,6 +6320,28 @@ func PutBucketPolicy_non_existing_bucket(s *S3Conf) error {
 	})
 }
 
+func PutBucketPolicy_empty_statement(s *S3Conf) error {
+	testName := "PutBucketPolicy_empty_statement"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		doc := `
+		{
+			"Statement": []
+		}
+		`
+
+		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+		_, err := s3client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+			Bucket: &bucket,
+			Policy: &doc,
+		})
+		cancel()
+		if err := checkApiErr(err, getMalformedPolicyError("Could not parse the policy: Statement is empty!")); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 func PutBucketPolicy_invalid_effect(s *S3Conf) error {
 	testName := "PutBucketPolicy_invalid_effect"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -6331,7 +6354,7 @@ func PutBucketPolicy_invalid_effect(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid effect: invalid_effect")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid effect: invalid_effect")); err != nil {
 			return err
 		}
 		return nil
@@ -6350,7 +6373,7 @@ func PutBucketPolicy_empty_actions_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6369,7 +6392,7 @@ func PutBucketPolicy_empty_actions_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("actions can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6388,7 +6411,7 @@ func PutBucketPolicy_invalid_action(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid action: ListObjects")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6407,7 +6430,7 @@ func PutBucketPolicy_unsupported_action(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported action: s3:PutLifecycleConfiguration")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6426,7 +6449,7 @@ func PutBucketPolicy_incorrect_action_wildcard_usage(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("invalid wildcard usage: s3:hello prefix is not in the supported actions list")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid action")); err != nil {
 			return err
 		}
 		return nil
@@ -6445,7 +6468,7 @@ func PutBucketPolicy_empty_principals_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6464,7 +6487,7 @@ func PutBucketPolicy_empty_principals_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6483,7 +6506,7 @@ func PutBucketPolicy_principals_aws_struct_empty_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6502,7 +6525,7 @@ func PutBucketPolicy_principals_aws_struct_empty_string_slice(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6521,7 +6544,7 @@ func PutBucketPolicy_principals_incorrect_wildcard_usage(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("principals should either contain * or user access keys")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
 			return err
 		}
 		return nil
@@ -6540,14 +6563,8 @@ func PutBucketPolicy_non_existing_principals(s *S3Conf) error {
 		})
 		cancel()
 
-		apiErr1 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_1", "a_rarely_existing_user_account_2"}))
-		apiErr2 := getMalformedPolicyError(fmt.Sprintf("user accounts don't exist: %v", []string{"a_rarely_existing_user_account_2", "a_rarely_existing_user_account_1"}))
-
-		err1 := checkApiErr(err, apiErr1)
-		err2 := checkApiErr(err, apiErr2)
-
-		if err1 != nil && err2 != nil {
-			return err1
+		if err := checkApiErr(err, getMalformedPolicyError("Invalid principal in policy")); err != nil {
+			return err
 		}
 
 		return nil
@@ -6566,7 +6583,7 @@ func PutBucketPolicy_empty_resources_string(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6585,7 +6602,7 @@ func PutBucketPolicy_empty_resources_array(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("resources can't be empty")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6605,7 +6622,7 @@ func PutBucketPolicy_invalid_resource_prefix(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6625,7 +6642,7 @@ func PutBucketPolicy_invalid_resource_with_starting_slash(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("invalid resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6644,10 +6661,10 @@ func PutBucketPolicy_duplicate_resource(s *S3Conf) error {
 			Policy: &doc,
 		})
 		cancel()
-
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("duplicate resource: %v", resource[1:len(resource)-1]))); err != nil {
+		if err != nil {
 			return err
 		}
+
 		return nil
 	})
 }
@@ -6665,7 +6682,7 @@ func PutBucketPolicy_incorrect_bucket_name(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError(fmt.Sprintf("incorrect bucket name in prefix-%v", bucket))); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Policy has invalid resource")); err != nil {
 			return err
 		}
 		return nil
@@ -6685,7 +6702,7 @@ func PutBucketPolicy_object_action_on_bucket_resource(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported object action 's3:PutObjectTagging' on the specified resources")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Action does not apply to any resource(s) in statement")); err != nil {
 			return err
 		}
 		return nil
@@ -6705,7 +6722,7 @@ func PutBucketPolicy_bucket_action_on_object_resource(s *S3Conf) error {
 		})
 		cancel()
 
-		if err := checkApiErr(err, getMalformedPolicyError("unsupported bucket action 's3:DeleteBucket' on the specified resources")); err != nil {
+		if err := checkApiErr(err, getMalformedPolicyError("Action does not apply to any resource(s) in statement")); err != nil {
 			return err
 		}
 		return nil

tests/remove_static.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+source ./tests/setup.sh
+source ./tests/util.sh
+
+delete_bucket_if_exists() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "delete_bucket_if_exists command missing command type, name"
+    return 1
+  fi
+  bucket_exists "$1" "$2" || local exists_result=$?
+  if [[ $exists_result -eq 2 ]]; then
+    log 2 "error checking if bucket exists"
+    return 1
+  fi
+  if [[ $exists_result -eq 1 ]]; then
+    log 5 "bucket '$2' doesn't exist, skipping"
+    return 0
+  fi
+  if ! delete_bucket_recursive "$1" "$2"; then
+    log 2 "error deleting bucket"
+    return 1
+  fi
+  log 5 "bucket '$2' successfully deleted"
+  return 0
+}
+
+if ! setup; then
+  log 2 "error starting versity to set up static buckets"
+  exit 1
+fi
+if ! delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME"; then
+  log 2 "error deleting static bucket one"
+elif ! delete_bucket_if_exists "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error deleting static bucket two"
+fi
+if ! teardown; then
+  log 2 "error stopping versity"
+fi

tests/run.sh

@@ -19,9 +19,6 @@ handle_param() {
           show_help
           exit 0
           ;;
-      -s|--static)
-          export RECREATE_BUCKETS=false
-          ;;
       s3|s3api|aws|s3cmd|mc|aws-user)
           set_command_type "$1"
           ;;
@@ -41,14 +38,6 @@ set_command_type() {
   export command_type
 }
 
-if [[ -z $RECREATE_BUCKETS ]]; then
-  export RECREATE_BUCKETS=true
-elif [[ $RECREATE_BUCKETS != true ]] && [[ $RECREATE_BUCKETS != false ]]; then
-  echo "Invalid RECREATE_BUCKETS value: $RECREATE_BUCKETS"
-  exit 1
-else
-  export RECREATE_BUCKETS=$RECREATE_BUCKETS
-fi
 while [[ "$#" -gt 0 ]]; do
   handle_param "$1"
   shift # past argument or value
@@ -59,13 +48,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
   exit 1
 fi
 
-if [[ $RECREATE_BUCKETS == false ]]; then
-  ./tests/setup_static.sh || exit_code=$?
-  if [[ exit_code -ne 0 ]]; then
-    exit 1
-  fi
-fi
-
 case $command_type in
   s3api|aws)
     echo "Running aws tests ..."

tests/run_all.sh

@@ -5,10 +5,6 @@ if [[ -z "$VERSITYGW_TEST_ENV" ]] && [[ $BYPASS_ENV_FILE != "true" ]]; then
   exit 1
 fi
 
-# shellcheck source=./tests/.env.default
-source "$VERSITYGW_TEST_ENV"
-export RECREATE_BUCKETS
-
 if ! ./tests/run.sh aws; then
   exit 1
 fi
@@ -21,7 +17,4 @@ fi
 if ! ./tests/run.sh mc; then
   exit 1
 fi
-if ! ./tests/run.sh user; then
-  exit 1
-fi
 exit 0

tests/setup_static.sh

@@ -1,7 +1,40 @@
 #!/bin/bash
 
 source ./tests/setup.sh
-setup
-aws --no-verify-ssl s3 mb s3://"$BUCKET_ONE_NAME"
-aws --no-verify-ssl s3 mb s3://"$BUCKET_TWO_NAME"
-teardown
+source ./tests/util.sh
+source ./tests/commands/create_bucket.sh
+
+create_bucket_if_not_exists() {
+  if [[ $# -ne 2 ]]; then
+    log 2 "create_bucket_if_not_exists command missing command type, name"
+    return 1
+  fi
+  bucket_exists "$1" "$2" || local exists_result=$?
+  if [[ $exists_result -eq 2 ]]; then
+    log 2 "error checking if bucket exists"
+    return 1
+  fi
+  if [[ $exists_result -eq 0 ]]; then
+    log 5 "bucket '$2' already exists, skipping"
+    return 0
+  fi
+  if ! create_bucket_object_lock_enabled "$2"; then
+    log 2 "error creating bucket"
+    return 1
+  fi
+  log 5 "bucket '$2' successfully created"
+  return 0
+}
+
+if ! setup; then
+  log 2 "error starting versity to set up static buckets"
+  exit 1
+fi
+if ! create_bucket_if_not_exists "s3api" "$BUCKET_ONE_NAME"; then
+  log 2 "error creating static bucket one"
+elif ! create_bucket_if_not_exists "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error creating static bucket two"
+fi
+if ! teardown; then
+  log 2 "error stopping versity"
+fi

tests/test_aws.sh

@@ -33,19 +33,17 @@ export RUN_USERS=true
 # abort-multipart-upload
 @test "test_abort_multipart_upload" {
   local bucket_file="bucket-file"
-  bucket_file_data="test file\n"
 
-  create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
-  [[ $created -eq 0 ]] || fail "Error creating test files"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  create_test_files "$bucket_file" || fail "error creating test files"
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
 
-  run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || abort_result=$?
-  [[ $abort_result -eq 0 ]] || fail "Abort failed"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
-  object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file" || exists=$?
-  [[ $exists -eq 1 ]] || fail "Upload file exists after abort"
+  run_then_abort_multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "abort failed"
+
+  if object_exists "aws" "$BUCKET_ONE_NAME" "$bucket_file"; then
+    fail "Upload file exists after abort"
+  fi
 
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
@@ -54,20 +52,15 @@ export RUN_USERS=true
 # complete-multipart-upload
 @test "test_complete_multipart_upload" {
   local bucket_file="bucket-file"
-  bucket_file_data="test file\n"
 
-  create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
-  [[ $created -eq 0 ]] || fail "Error creating test files"
-  setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
+  create_test_files "$bucket_file" || fail "error creating test files"
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
 
-  multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || upload_result=$?
-  [[ $upload_result -eq 0 ]] || fail "Error performing multipart upload"
+  setup_bucket "aws" "$BUCKET_ONE_NAME" || fail "failed to create bucket '$BUCKET_ONE_NAME'"
 
-  copy_file "s3://$BUCKET_ONE_NAME/$bucket_file" "$test_file_folder/$bucket_file-copy"
-  compare_files "$test_file_folder/$bucket_file-copy" "$test_file_folder"/$bucket_file || compare_result=$?
-  [[ $compare_result -eq 0 ]] || fail "Files do not match"
+  multipart_upload "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 || fail "error performing multipart upload"
+
+  download_and_compare_file "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error downloading and comparing file"
 
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
@@ -91,7 +84,6 @@ export RUN_USERS=true
 # create-multipart-upload
 @test "test_create_multipart_upload_properties" {
   local bucket_file="bucket-file"
-  local bucket_file_data="test file\n"
 
   local expected_content_type="application/zip"
   local expected_meta_key="testKey"
@@ -105,25 +97,30 @@ export RUN_USERS=true
   os_name="$(uname)"
   if [[ "$os_name" == "Darwin" ]]; then
     now=$(date -u +"%Y-%m-%dT%H:%M:%S")
-    five_seconds_later=$(date -j -v +10S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
+    later=$(date -j -v +15S -f "%Y-%m-%dT%H:%M:%S" "$now" +"%Y-%m-%dT%H:%M:%S")
   else
     now=$(date +"%Y-%m-%dT%H:%M:%S")
-    five_seconds_later=$(date -d "$now 10 seconds" +"%Y-%m-%dT%H:%M:%S")
+    later=$(date -d "$now 15 seconds" +"%Y-%m-%dT%H:%M:%S")
   fi
 
   create_test_files "$bucket_file" || fail "error creating test file"
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
 
-  delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
-  create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
+  delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
+  # in static bucket config, bucket will still exist
+  bucket_exists "s3api" "$BUCKET_ONE_NAME" || local exists_result=$?
+  [[ $exists_result -ne 2 ]] || fail "error checking for bucket existence"
+  if [[ $exists_result -eq 1 ]]; then
+    create_bucket_object_lock_enabled "$BUCKET_ONE_NAME" || fail "error creating bucket"
+  fi
 
-  log 5 "$five_seconds_later"
+  log 5 "LATER: $later"
   multipart_upload_with_params "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder"/"$bucket_file" 4 \
     "$expected_content_type" \
     "{\"$expected_meta_key\": \"$expected_meta_val\"}" \
     "$expected_hold_status" \
     "$expected_retention_mode" \
-    "$five_seconds_later" \
+    "$later" \
     "$expected_tag_key=$expected_tag_val" || fail "error performing multipart upload"
 
   head_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting metadata"
@@ -139,7 +136,7 @@ export RUN_USERS=true
   retention_mode=$(echo "$raw_metadata" | jq -r ".ObjectLockMode")
   [[ $retention_mode == "$expected_retention_mode" ]] || fail "retention mode mismatch ($retention_mode, $expected_retention_mode)"
   retain_until_date=$(echo "$raw_metadata" | jq -r ".ObjectLockRetainUntilDate")
-  [[ $retain_until_date == "$five_seconds_later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"
+  [[ $retain_until_date == "$later"* ]] || fail "retention date mismatch ($retain_until_date, $five_seconds_later)"
 
   get_object_tagging "aws" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error getting tagging"
   log 5 "tags: $tags"
@@ -154,7 +151,7 @@ export RUN_USERS=true
   get_object "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "error getting object"
   compare_files "$test_file_folder/$bucket_file" "$test_file_folder/$bucket_file-copy" || fail "files not equal"
 
-  sleep 10
+  sleep 15
 
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
   delete_test_files $bucket_file
@@ -226,18 +223,16 @@ export RUN_USERS=true
   delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
 }
 
-#@test "test_get_object_invalid_range" {
-#  bucket_file="bucket_file"
-#
-#  create_test_files "$bucket_file" || local created=$?
-#  [[ $created -eq 0 ]] || fail "Error creating test files"
-#  setup_bucket "s3api" "$BUCKET_ONE_NAME" || local setup_result=$?
-#  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
-#  put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error putting object"
-#  get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=0-0" "$test_file_folder/$bucket_file-range" || local get_result=$?
-#  [[ $get_result -ne 0 ]] || fail "Get object with zero range returned no error"
-#}
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "aws"
+}
 
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging - test_set_get_delete_bucket_tags
+
+# get-object
 @test "test_get_object_full_range" {
   bucket_file="bucket_file"
 
@@ -251,6 +246,18 @@ export RUN_USERS=true
   [[ "$(cat "$test_file_folder/$bucket_file-range")" == "9" ]] || fail "byte range not copied properly"
 }
 
+#@test "test_get_object_invalid_range" {
+#  bucket_file="bucket_file"
+#
+#  create_test_files "$bucket_file" || local created=$?
+#  [[ $created -eq 0 ]] || fail "Error creating test files"
+#  setup_bucket "s3api" "$BUCKET_ONE_NAME" || local setup_result=$?
+#  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+#  put_object "s3api" "$test_file_folder/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" || fail "error putting object"
+#  get_object_with_range "$BUCKET_ONE_NAME" "$bucket_file" "bytes=0-0" "$test_file_folder/$bucket_file-range" || local get_result=$?
+#  [[ $get_result -ne 0 ]] || fail "Get object with zero range returned no error"
+#}
+
 @test "test_put_object" {
   bucket_file="bucket_file"
 
@@ -417,7 +424,7 @@ legal_hold_retention_setup() {
     return 1
   fi
 
-  delete_bucket_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
+  delete_bucket_or_contents_if_exists "s3api" "$BUCKET_ONE_NAME" || fail "error deleting bucket, or checking for existence"
   create_user_if_nonexistent "$1" "$2" "user" || fail "error creating user if nonexistent"
   create_test_files "$3" || fail "error creating test files"
 
@@ -540,11 +547,10 @@ legal_hold_retention_setup() {
 # test multi-part upload list parts command
 @test "test-multipart-upload-list-parts" {
   local bucket_file="bucket-file"
-  local bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
   [[ $created -eq 0 ]] || fail "Error creating test files"
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -623,11 +629,10 @@ legal_hold_retention_setup() {
 
 @test "test-multipart-upload-from-bucket" {
   local bucket_file="bucket-file"
-  bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
   [[ $created -eq 0 ]] || fail "Error creating test files"
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/$bucket_file
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
   setup_bucket "aws" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -708,6 +713,418 @@ legal_hold_retention_setup() {
   delete_test_files $folder_name
 }
 
+#@test "test_put_policy_no_version" {
+#  policy_file="policy_file"
+#
+#  create_test_files "$policy_file" || fail "error creating policy file"
+#
+#  effect="Allow"
+#  principal="*"
+#  action="s3:GetObject"
+#  resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
+#
+#  cat <<EOF > "$test_file_folder"/$policy_file
+#    {
+#      "Statement": [
+#        {
+#           "Effect": "$effect",
+#           "Principal": "$principal",
+#           "Action": "$action",
+#           "Resource": "$resource"
+#        }
+#      ]
+#    }
+#EOF
+#
+#    setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+#
+#    check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty"
+#
+#    put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+#
+#    get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "unable to retrieve policy"
+#}
+
+@test "test_put_policy_invalid_action" {
+  policy_file="policy_file"
+
+  create_test_files "$policy_file" || fail "error creating policy file"
+
+  effect="Allow"
+  principal="*"
+  action="s3:GetObjectt"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"
+
+  cat <<EOF > "$test_file_folder"/$policy_file
+    {
+      "Statement": [
+        {
+           "Effect": "$effect",
+           "Principal": "$principal",
+           "Action": "$action",
+           "Resource": "$resource"
+        }
+      ]
+    }
+EOF
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+
+  check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty"
+
+  if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file"; then
+    fail "put succeeded despite malformed policy"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$put_bucket_policy_error" == *"MalformedPolicy"*"invalid action"* ]] || fail "invalid policy error: $put_bucket_policy_error"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$policy_file"
+}
+
+@test "test_policy_get_object_with_user" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file"
+  username="ABCDEFG"
+  password="HIJKLMN"
+  test_file="test_file"
+
+  create_test_files "$test_file" "$policy_file" || fail "error creating policy file"
+  echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file"
+
+  effect="Allow"
+  principal="$username"
+  action="s3:GetObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+
+  setup_policy_with_single_statement "$test_file_folder/$policy_file" "2012-10-17" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object"
+
+  if ! check_for_empty_policy "s3api" "$BUCKET_ONE_NAME"; then
+    delete_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error deleting policy"
+    check_for_empty_policy "s3api" "$BUCKET_ONE_NAME" || fail "policy not empty after deletion"
+  fi
+
+  if put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file"; then
+    fail "put succeeded despite invalid username"
+  fi
+  [[ "$put_bucket_policy_error" == *"MalformedPolicy"* ]] || fail "invalid policy error: $put_bucket_policy_error"
+
+#  cat <<EOF > "$test_file_folder"/acl_file
+#{
+#  "Grants": [
+#    {
+#      "Grantee": {
+#        "ID": "ABCDEFG",
+#        "Type": "CanonicalUser"
+#      },
+#      "Permission": "READ"
+#    }
+#  ],
+#  "Owner": {
+#    "ID": "$AWS_ACCESS_KEY_ID"
+#  }
+#}
+#EOF
+#
+  #put_bucket_acl "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/acl_file" || fail "error putting acl"
+  #put_bucket_canned_acl "$BUCKET_ONE_NAME" "public-read-write" || fail "error putting acl"
+
+  create_user "$username" "$password" "user" || fail "error creating user"
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then
+    fail "get object with user succeeded despite lack of permissions"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+  #get_bucket_policy "s3api" "$BUCKET_ONE_NAME" || fail "error getting bucket policy"
+  #log 5 "$bucket_policy"
+  get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password" || fail "error getting object after permissions"
+  compare_files "$test_file_folder/$test_file" "$test_file_folder/$test_file-copy" || fail "files not equal"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_policy_get_object_specific_file" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file"
+  test_file="test_file"
+  test_file_two="test_file_two"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_files "$policy_file" "$test_file" "$test_file_two" || fail "error creating policy file"
+  echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_file"
+  echo "$BATS_TEST_NAME-2" >> "$test_file_folder/$test_file_two"
+
+  effect="Allow"
+  principal="$username"
+  action="s3:GetObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/test_file"
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+
+  put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object"
+  put_object "s3api" "$test_file_folder/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two" || fail "error copying object"
+
+  get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password" || fail "error getting object after permissions"
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$test_file_folder/$test_file_two-copy" "$username" "$password"; then
+    fail "get object with user succeeded despite lack of permissions"
+  fi
+  # shellcheck disable=SC2154
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_policy_get_object_file_wildcard" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file_one"
+  policy_file_two="policy_file_two"
+  policy_file_three="policy_fil"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_files "$policy_file" "$policy_file_two" "$policy_file_three" || fail "error creating policy file"
+  echo "$BATS_TEST_NAME" >> "$test_file_folder/$policy_file"
+
+  effect="Allow"
+  principal="$username"
+  action="s3:GetObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/policy_file*"
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user account"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+
+  put_object "s3api" "$test_file_folder/$policy_file" "$BUCKET_ONE_NAME" "$policy_file" || fail "error copying object one"
+  put_object "s3api" "$test_file_folder/$policy_file_two" "$BUCKET_ONE_NAME" "$policy_file_two" || fail "error copying object two"
+  put_object "s3api" "$test_file_folder/$policy_file_three" "$BUCKET_ONE_NAME" "$policy_file_three" || fail "error copying object three"
+
+  get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$policy_file" "$test_file_folder/$policy_file" "$username" "$password" || fail "error getting object one after permissions"
+  get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$policy_file_two" "$test_file_folder/$policy_file_two" "$username" "$password" || fail "error getting object two after permissions"
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$policy_file_three" "$test_file_folder/$policy_file_three" "$username" "$password"; then
+    fail "get object three with user succeeded despite lack of permissions"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+}
+
+@test "test_policy_get_object_folder_wildcard" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file"
+  test_folder="test_folder"
+  test_file="test_file"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_folder "$test_folder" || fail "error creating test folder"
+  create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file"
+  echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_folder/$test_file"
+
+  effect="Allow"
+  principal="$username"
+  action="s3:GetObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+
+  put_object "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" || fail "error copying object to bucket"
+
+  download_and_compare_file_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_file_folder/$test_file-copy" "$username" "$password" || fail "error downloading and comparing file"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_folder/$test_file" "$policy_file"
+}
+
+@test "test_policy_allow_deny" {
+  policy_file="policy_file"
+  test_file="test_file"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_files "$policy_file" "$test_file" || fail "error creating policy file"
+
+  principal="$username"
+  action="s3:GetObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_file"
+
+  cat <<EOF > "$test_file_folder"/$policy_file
+    {
+      "Statement": [
+        {
+           "Effect": "Deny",
+           "Principal": "$principal",
+           "Action": "$action",
+           "Resource": "$resource"
+        },
+        {
+           "Effect": "Allow",
+           "Principal": "$principal",
+           "Action": "$action",
+           "Resource": "$resource"
+        }
+      ]
+    }
+EOF
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user"
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+  put_object "s3api" "$test_file_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" || fail "error copying object to bucket"
+
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file" "$test_file_folder/$test_file-copy" "$username" "$password"; then
+    fail "able to get object despite deny statement"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file" "$test_file-copy" "$policy_file"
+}
+
+@test "test_policy_deny" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file"
+  test_file_one="test_file_one"
+  test_file_two="test_file_two"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_files "$test_file_one" "$test_file_two" "$policy_file" || fail "error creating policy file, test file"
+
+  cat <<EOF > "$test_file_folder"/$policy_file
+{
+  "Statement": [
+    {
+       "Effect": "Deny",
+       "Principal": "$username",
+       "Action": "s3:GetObject",
+       "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/$test_file_two"
+    },
+    {
+       "Effect": "Allow",
+       "Principal": "$username",
+       "Action": "s3:GetObject",
+       "Resource": "arn:aws:s3:::$BUCKET_ONE_NAME/*"
+    }
+  ]
+}
+EOF
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  log 5 "Policy: $(cat "$test_file_folder/$policy_file")"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+  put_object "s3api" "$test_file_folder/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_one" || fail "error copying object one"
+  put_object "s3api" "$test_file_folder/$test_file_one" "$BUCKET_ONE_NAME" "$test_file_two" || fail "error copying object two"
+  get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_one" "$test_file_folder/$test_file_one-copy" "$username" "$password" || fail "error getting object"
+  if get_object_with_user "s3api" "$BUCKET_ONE_NAME" "$test_file_two" "$test_file_folder/$test_file_two-copy" "$username" "$password"; then
+    fail "able to get object despite deny statement"
+  fi
+  [[ "$get_object_error" == *"Access Denied"* ]] || fail "invalid get object error: $get_object_error"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_file_one" "$test_file_two" "$test_file_one-copy" "$test_file_two-copy" "$policy_file"
+}
+
+@test "test_policy_put_wildcard" {
+  # TODO (https://github.com/versity/versitygw/issues/637)
+  if [[ $RECREATE_BUCKETS == "false" ]]; then
+    return 0
+  fi
+
+  policy_file="policy_file"
+  test_folder="test_folder"
+  test_file="test_file"
+  username="ABCDEFG"
+  password="HIJKLMN"
+
+  create_test_folder "$test_folder" || fail "error creating test folder"
+  create_test_files "$test_folder/$test_file" "$policy_file" || fail "error creating policy file, test file"
+  echo "$BATS_TEST_NAME" >> "$test_file_folder/$test_folder/$test_file"
+
+  effect="Allow"
+  principal="$username"
+  action="s3:PutObject"
+  resource="arn:aws:s3:::$BUCKET_ONE_NAME/$test_folder/*"
+
+  if user_exists "$username"; then
+    delete_user "$username" || fail "failed to delete user '$username'"
+  fi
+  create_user "$username" "$password" "user" || fail "error creating user"
+
+  setup_bucket "s3api" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
+  log 5 "Policy: $(cat "$test_file_folder/$policy_file")"
+  setup_policy_with_single_statement "$test_file_folder/$policy_file" "dummy" "$effect" "$principal" "$action" "$resource" || fail "failed to set up policy"
+  put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$test_file_folder/$policy_file" || fail "error putting policy"
+  if put_object_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password"; then
+    fail "able to put object despite not being allowed"
+  fi
+  [[ "$put_object_error" == *"Access Denied"* ]] || fail "invalid put object error: $put_object_error"
+  put_object_with_user "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$username" "$password" || fail "error putting file despite policy permissions"
+  download_and_compare_file "s3api" "$test_file_folder/$test_folder/$test_file" "$BUCKET_ONE_NAME" "$test_folder/$test_file" "$test_file_folder/$test_file-copy" || fail "files don't match"
+  delete_bucket_or_contents "aws" "$BUCKET_ONE_NAME"
+  delete_test_files "$test_folder/$test_file" "$test_file-copy" "$policy_file"
+}
+
+# ensure that lists of files greater than a size of 1000 (pagination) are returned properly
+#@test "test_list_objects_file_count" {
+#  test_common_list_objects_file_count "aws"
+#}
+
+#@test "test_filename_length" {
+#  file_name=$(printf "%0.sa" $(seq 1 1025))
+#  echo "$file_name"
+
+
 # ensure that lists of files greater than a size of 1000 (pagination) are returned properly
 #@test "test_list_objects_file_count" {
 #  test_common_list_objects_file_count "aws"
@@ -749,7 +1166,6 @@ legal_hold_retention_setup() {
 }
 
 @test "test_add_object_metadata" {
-
   object_one="object-one"
   test_key="x-test-data"
   test_value="test-value"
@@ -774,7 +1190,3 @@ legal_hold_retention_setup() {
   [[ $value == "\"$test_value\"" ]] || fail "values doesn't match (expected $value, actual \"$test_value\")"
 }
 
-
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "aws"
-}

tests/test_common.sh

@@ -137,28 +137,24 @@ test_common_put_get_object() {
   fi
 
   local object_name="test-object"
-  create_test_files "$object_name" || local create_result=$?
-  [[ $create_result -eq 0 ]] || fail "Error creating test file"
+
+  create_test_files "$object_name" || fail "error creating test file"
   echo "test data" > "$test_file_folder"/"$object_name"
 
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local setup_result=$?
-  [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error setting up bucket"
 
-  put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || local copy_result=$?
-  [[ $copy_result -eq 0 ]] || fail "Failed to add object to bucket"
-  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_one=$?
-  [[ $exists_result_one -eq 0 ]] || fail "Object not added to bucket"
+  if [[ $1 == 's3' ]]; then
+    copy_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to add object to bucket"
+  else
+    put_object "$1" "$test_file_folder/$object_name" "$BUCKET_ONE_NAME" "$object_name" || fail "failed to add object to bucket"
+  fi
+  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || fail "object not added to bucket"
 
-  get_object "$1" "$BUCKET_ONE_NAME" "$object_name" "$test_file_folder/${object_name}_copy" || local delete_result=$?
-  [[ $delete_result -eq 0 ]] || fail "Failed to delete object"
-  object_exists "$1" "$BUCKET_ONE_NAME" "$object_name" || local exists_result_two=$?
-  [[ $exists_result_two -eq 1 ]] || fail "Object not removed from bucket"
-
-  compare_files "$test_file_folder"/"$object_name" "$test_file_folder/${object_name}_copy" || compare_result=$?
-  [[ $compare_result -ne 0 ]] || fail "objects are different"
+  get_object "$1" "$BUCKET_ONE_NAME" "$object_name" "$test_file_folder/${object_name}_copy" || fail "failed to get object"
+  compare_files "$test_file_folder"/"$object_name" "$test_file_folder/${object_name}_copy" || fail "objects are different"
 
   delete_bucket_or_contents "$1" "$BUCKET_ONE_NAME"
-  delete_test_files "$test_file_folder/$object_name" "$test_file_folder/${object_name}_copy"
+  delete_test_files "$object_name" "${object_name}_copy"
 }
 
 test_common_get_set_versioning() {
@@ -349,10 +345,9 @@ test_common_presigned_url_utf8_chars() {
 
   local bucket_file="my-$%^&*;"
   local bucket_file_copy="bucket-file-copy"
-  bucket_file_data="test file\n"
 
   create_test_files "$bucket_file" || local created=$?
-  printf "%s" "$bucket_file_data" > "$test_file_folder"/"$bucket_file"
+  dd if=/dev/urandom of="$test_file_folder/$bucket_file" bs=5M count=1 || fail "error creating test file"
   setup_bucket "$1" "$BUCKET_ONE_NAME" || local result=$?
   [[ $result -eq 0 ]] || fail "Failed to create bucket '$BUCKET_ONE_NAME'"
 
@@ -440,12 +435,10 @@ test_common_get_bucket_location() {
 
 test_common_put_bucket_acl() {
   [[ $# -eq 1 ]] || fail "test common put bucket acl missing command type"
-  setup_bucket "$1" "$BUCKET_ONE_NAME" || local created=$?
-  [[ $created -eq 0 ]] || fail "Error creating bucket"
+  setup_bucket "$1" "$BUCKET_ONE_NAME" || fail "error creating bucket"
 
   if ! user_exists "ABCDEFG"; then
-    create_user "ABCDEFG" "HIJKLMN" user || create_result=$?
-    [[ $create_result -eq 0 ]] || fail "Error creating user"
+    create_user "ABCDEFG" "HIJKLMN" user || fail "error creating user"
   fi
 
   get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
@@ -480,11 +473,15 @@ cat <<EOF > "$test_file_folder"/"$acl_file"
   }
 EOF
 
-  put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+  log 6 "before 1st put acl"
+  if [[ $1 == 's3api' ]] || [[ $1 == 'aws' ]]; then
+    put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || fail "error putting first acl"
+  else
+    put_bucket_acl "$1" "$BUCKET_ONE_NAME" "ABCDEFG" || fail "error putting first acl"
+  fi
 
   get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+  [[ $result -eq 0 ]] || fail "Error retrieving second acl"
 
   log 5 "Acls after 1st put: $acl"
   public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants[0]')
@@ -509,10 +506,10 @@ cat <<EOF > "$test_file_folder"/"$acl_file"
 EOF
 
   put_bucket_acl "$1" "$BUCKET_ONE_NAME" "$test_file_folder"/"$acl_file" || local put_result=$?
-  [[ $put_result -eq 0 ]] || fail "Error putting acl"
+  [[ $put_result -eq 0 ]] || fail "Error putting second acl"
 
   get_bucket_acl "$1" "$BUCKET_ONE_NAME" || local result=$?
-  [[ $result -eq 0 ]] || fail "Error retrieving acl"
+  [[ $result -eq 0 ]] || fail "Error retrieving second acl"
 
   log 5 "Acls after 2nd put: $acl"
   public_grants=$(echo "$acl" | grep -v "InsecureRequestWarning" | jq -r '.Grants')

tests/test_mc.sh

@@ -38,13 +38,30 @@ export RUN_MC=true
 
 # delete-object - put-object tests
 
-# delete-objects - test setup/teardown
-
 # delete-object-tagging
 @test "test_delete_object_tagging" {
   test_common_delete_object_tagging "mc"
 }
 
+# delete-objects - test setup/teardown
+
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "mc"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-bucket-tagging
+@test "test_set_get_object_tags_mc" {
+  test_common_set_get_object_tags "mc"
+}
+
+# get-object
+@test "test_put_get_object" {
+  test_common_put_get_object "mc"
+}
+
 @test "test_put_object-with-data-mc" {
   test_common_put_object_with_data "mc"
 }
@@ -61,9 +78,6 @@ export RUN_MC=true
   test_common_list_objects "mc"
 }
 
-@test "test_set_get_object_tags_mc" {
-  test_common_set_get_object_tags "mc"
-}
 
 @test "test_presigned_url_utf8_chars_mc" {
   test_common_presigned_url_utf8_chars "mc"
@@ -104,6 +118,3 @@ export RUN_MC=true
 }
 
 
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "mc"
-}

tests/test_s3.sh

@@ -23,6 +23,11 @@ source ./tests/test_common.sh
 
 # delete-objects - tested with recursive bucket delete
 
+# get-object
+@test "test_copy_get_object" {
+  test_common_put_get_object "s3"
+}
+
 @test "test_put_object" {
   test_common_put_object_no_data "s3"
 }
@@ -34,7 +39,3 @@ source ./tests/test_common.sh
 @test "test_list_objects_file_count" {
   test_common_list_objects_file_count "s3"
 }
-
-@test "test_put_get_object" {
-  test_common_put_get_object "s3"
-}

tests/test_s3cmd.sh

@@ -10,6 +10,7 @@ source ./tests/commands/get_bucket_policy.sh
 source ./tests/commands/put_bucket_policy.sh
 
 export RUN_S3CMD=true
+export RUN_USERS=true
 
 # complete-multipart-upload
 @test "test_complete_multipart_upload" {
@@ -52,6 +53,26 @@ export RUN_S3CMD=true
 
 # get-bucket-acl - test_put_bucket_acl
 
+# get-bucket-location
+@test "test_get_bucket_location" {
+  test_common_get_bucket_location "s3cmd"
+}
+
+# get-bucket-policy - test_get_put_delete_bucket_policy
+
+# get-object
+@test "test_put_get_object" {
+  test_common_put_get_object "s3cmd"
+}
+
+@test "test_put_object_with_data" {
+  test_common_put_object_with_data "s3cmd"
+}
+
+@test "test_put_object_no_data" {
+  test_common_put_object_no_data "s3cmd"
+}
+
 #@test "test_put_bucket_acl" {
 #  test_common_put_bucket_acl "s3cmd"
 #}
@@ -73,7 +94,6 @@ export RUN_S3CMD=true
   test_common_list_objects_file_count "s3cmd"
 }
 
-
 @test "test_get_bucket_info_s3cmd" {
   setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
   [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
@@ -82,15 +102,6 @@ export RUN_S3CMD=true
   delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
 }
 
-# put-object
-@test "test_put_object_with_data" {
-  test_common_put_object_with_data "s3cmd"
-}
-
-@test "test_put_object_no_data" {
-  test_common_put_object_no_data "s3cmd"
-}
-
 @test "test_get_bucket_info_doesnt_exist_s3cmd" {
   setup_bucket "s3cmd" "$BUCKET_ONE_NAME" || local setup_result=$?
   [[ $setup_result -eq 0 ]] || fail "error setting up bucket"
@@ -100,6 +111,3 @@ export RUN_S3CMD=true
   delete_bucket_or_contents "s3cmd" "$BUCKET_ONE_NAME"
 }
 
-@test "test_get_bucket_location" {
-  test_common_get_bucket_location "s3cmd"
-}

tests/util.sh

@@ -8,12 +8,14 @@ source ./tests/commands/complete_multipart_upload.sh
 source ./tests/commands/create_multipart_upload.sh
 source ./tests/commands/create_bucket.sh
 source ./tests/commands/delete_bucket.sh
+source ./tests/commands/delete_bucket_policy.sh
 source ./tests/commands/delete_object.sh
 source ./tests/commands/get_bucket_tagging.sh
 source ./tests/commands/get_object_tagging.sh
 source ./tests/commands/head_bucket.sh
 source ./tests/commands/head_object.sh
 source ./tests/commands/list_objects.sh
+source ./tests/commands/put_bucket_acl.sh
 source ./tests/commands/upload_part_copy.sh
 
 # recursively delete an AWS bucket
@@ -144,26 +146,34 @@ bucket_exists() {
 # return:  0 for success, 1 for failure
 delete_bucket_or_contents() {
   if [ $# -ne 2 ]; then
-    echo "delete bucket or contents function requires command type, bucket name"
+    log 2 "delete bucket or contents function requires command type, bucket name"
     return 1
   fi
   if [[ $RECREATE_BUCKETS == "false" ]]; then
-    delete_bucket_contents "$1" "$2" || local delete_result=$?
-    if [[ $delete_result -ne 0 ]]; then
-      echo "error deleting bucket contents"
+    if ! delete_bucket_contents "$1" "$2"; then
+      log 2 "error deleting bucket contents"
       return 1
     fi
+    if ! delete_bucket_policy "s3api" "$2"; then
+      log 2 "error deleting bucket policies"
+      return 1
+    fi
+    if ! put_bucket_canned_acl "$2" "private"; then
+      log 2 "error resetting bucket ACLs"
+      return 1
+    fi
+    log 5 "bucket contents, policy, ACL deletion success"
     return 0
   fi
-  delete_bucket_recursive "$1" "$2" || local delete_result=$?
-  if [[ $delete_result -ne 0 ]]; then
-    echo "Bucket deletion error"
+  if ! delete_bucket_recursive "$1" "$2"; then
+    log 2 "Bucket deletion error"
     return 1
   fi
+  log 5 "bucket deletion success"
   return 0
 }
 
-delete_bucket_if_exists() {
+delete_bucket_or_contents_if_exists() {
   if [ $# -ne 2 ]; then
     log 2 "bucket creation function requires command type, bucket name"
     return 1
@@ -180,10 +190,8 @@ delete_bucket_if_exists() {
       log 2 "error deleting bucket or contents"
       return 1
     fi
-    #if [[ $RECREATE_BUCKETS == "false" ]]; then
     log 5 "bucket and/or bucket data deletion success"
     return 0
-    #fi
   fi
   if [[ $RECREATE_BUCKETS == "false" ]]; then
     log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
@@ -201,19 +209,22 @@ setup_bucket() {
     log 2 "bucket creation function requires command type, bucket name"
     return 1
   fi
-  delete_bucket_if_exists "$1" "$2" || local delete_bucket_result=$?
+  delete_bucket_or_contents_if_exists "$1" "$2" || local delete_bucket_result=$?
   if [[ $delete_bucket_result -ne 0 ]]; then
     log 2 "error deleting bucket, or checking for bucket existence"
     return 1
   fi
   local create_result
   log 5 "util.setup_bucket: command type: $1, bucket name: $2"
-  create_bucket "$1" "$2" || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    log 2 "Error creating bucket"
-    return 1
+  if [[ $RECREATE_BUCKETS == "true" ]]; then
+    if ! create_bucket "$1" "$2"; then
+      log 2 "Error creating bucket"
+      return 1
+    fi
+    log 5 "bucket creation success"
+  else
+    log 5 "skipping bucket re-creation"
   fi
-  log 5 "Bucket creation success"
   return 0
 }
 

tests/util_file.sh

@@ -164,3 +164,29 @@ create_test_file_count() {
   fi
   return 0
 }
+
+download_and_compare_file() {
+  if [[ $# -ne 5 ]]; then
+    log 2 "'download and compare file' requires command type, original file, bucket, key, local file"
+    return 1
+  fi
+  download_and_compare_file_with_user "$1" "$2" "$3" "$4" "$5" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
+  return "$?"
+}
+
+download_and_compare_file_with_user() {
+  if [[ $# -ne 7 ]]; then
+    log 2 "'download and compare file with user' command requires command type, original file, bucket, key, local file, user, password"
+    return 1
+  fi
+  if ! get_object_with_user "$1" "$3" "$4" "$5" "$6" "$7"; then
+    log 2 "error retrieving file"
+    return 1
+  fi
+  log 5 "files: $2, $5"
+  if ! compare_files "$2" "$5"; then
+    log 2 "files don't match"
+    return 1
+  fi
+  return 0
+}

tests/util_policy.sh

@@ -6,23 +6,44 @@ check_for_empty_policy() {
     return 1
   fi
 
-  local get_result=0
-  get_bucket_policy "$1" "$2" || get_result=$?
-  if [[ $get_result -ne 0 ]]; then
-    echo "error getting bucket policy"
+  if ! get_bucket_policy "$1" "$2"; then
+    log 2 "error getting bucket policy"
     return 1
   fi
+  # shellcheck disable=SC2154
+  log 5 "bucket policy: $bucket_policy"
 
   # shellcheck disable=SC2154
   if [[ $bucket_policy == "" ]]; then
     return 0
   fi
 
-  policy=$(echo "$bucket_policy" | jq -r '.Policy')
-  statement=$(echo "$policy" | jq -r '.Statement[0]')
+  #policy=$(echo "$bucket_policy" | jq -r '.Policy')
+  statement=$(echo "$bucket_policy" | jq -r '.Statement[0]')
+  log 5 "statement: $statement"
   if [[ "" != "$statement" ]] && [[ "null" != "$statement" ]]; then
     echo "policy should be empty (actual value: '$statement')"
     return 1
   fi
   return 0
-}
+}
+
+setup_policy_with_single_statement() {
+  if [[ $# -ne 6 ]]; then
+    "'setup single policy' command requires file, version, effect, principal, action, resource"
+  fi
+  cat <<EOF > "$1"
+{
+  "Version": "$2",
+  "Statement": [
+    {
+       "Effect": "$3",
+       "Principal": "$4",
+       "Action": "$5",
+       "Resource": "$6"
+    }
+  ]
+}
+EOF
+log 5 "$(cat "$1")"
+}

tests/util_users.sh

@@ -2,12 +2,11 @@
 
 create_user() {
   if [[ $# -ne 3 ]]; then
-    echo "create user command requires user ID, key, and role"
+    log 2 "create user command requires user ID, key, and role"
     return 1
   fi
-  create_user_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3" || create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating user: $error"
+  if ! create_user_with_user "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$1" "$2" "$3"; then
+    log 2 "error creating user"
     return 1
   fi
   return 0
@@ -28,12 +27,11 @@ create_user_if_nonexistent() {
 
 create_user_with_user() {
   if [[ $# -ne 5 ]]; then
-    echo "create user with user command requires creator ID, key, and new user ID, key, and role"
+    log 2 "create user with user command requires creator ID, key, and new user ID, key, and role"
     return 1
   fi
-  error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5") || local create_result=$?
-  if [[ $create_result -ne 0 ]]; then
-    echo "error creating user: $error"
+  if ! error=$($VERSITY_EXE admin --allow-insecure --access "$1" --secret "$2" --endpoint-url "$AWS_ENDPOINT_URL" create-user --access "$3" --secret "$4" --role "$5" 2>&1); then
+    log 2 "error creating user: $error"
     return 1
   fi
   return 0

tests/versity.sh

@@ -28,8 +28,10 @@ start_versity_process() {
     return 1
   fi
   eval versitygw_pid_"$1"=$!
-  process_info="Versity process $1, PID $!"
-  echo "$process_info" >> "$VERSITY_LOG_FILE"
+  if [ -n "$VERSITY_LOG_FILE" ]; then
+    process_info="Versity process $1, PID $!"
+    echo "$process_info" >> "$VERSITY_LOG_FILE"
+  fi
   log 4 "$process_info"
   local pid
   eval pid=\$versitygw_pid_"$1"
@@ -72,6 +74,30 @@ run_versity_app_posix() {
   return 0
 }
 
+run_versity_app_scoutfs() {
+  if [[ $# -ne 3 ]]; then
+    echo "run versity app w/scoutfs command requires access ID, secret key, process number"
+    return 1
+  fi
+  base_command=("$VERSITY_EXE" --access="$1" --secret="$2" --region="$AWS_REGION"  --iam-dir="$USERS_FOLDER")
+  if [ -n "$CERT" ] && [ -n "$KEY" ]; then
+    base_command+=(--cert "$CERT" --key "$KEY")
+  fi
+  if [ -n "$PORT" ]; then
+    base_command+=(--port ":$PORT")
+  fi
+  base_command+=(scoutfs "$LOCAL_FOLDER")
+  export base_command
+
+  local versity_result
+  start_versity_process "$3" || versity_result=$?
+  if [[ $versity_result -ne 0 ]]; then
+    echo "error starting versity process"
+    return 1
+  fi
+  return 0
+}
+
 run_versity_app_s3() {
   if [[ $# -ne 1 ]]; then
     log 2 "run versity app w/s3 command requires process number"
@@ -102,6 +128,12 @@ run_versity_app() {
       log 2 "error starting versity app"
       return 1
     fi
+  elif [[ $BACKEND == 'scoutfs' ]]; then
+    run_versity_app_scoutfs "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1" || result_one=$?
+    if [[ $result_one -ne 0 ]]; then
+      echo "error starting versity app"
+      return 1
+    fi
   elif [[ $BACKEND == 's3' ]]; then
     if ! run_versity_app_posix "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "1"; then
       log 2 "error starting versity app"