diff --git a/auth/acl.go b/auth/acl.go new file mode 100644 index 0000000..c1dfc6e --- /dev/null +++ b/auth/acl.go @@ -0,0 +1,242 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" +) + +type ACL struct { + ACL types.BucketCannedACL + Owner string + Grantees []Grantee +} + +type Grantee struct { + Permission types.Permission + Access string +} + +type GetBucketAclOutput struct { + Owner *types.Owner + AccessControlList AccessControlList +} + +type AccessControlList struct { + Grants []types.Grant +} + +func ParseACL(data []byte) (ACL, error) { + if len(data) == 0 { + return ACL{}, nil + } + + var acl ACL + if err := json.Unmarshal(data, &acl); err != nil { + return acl, fmt.Errorf("parse acl: %w", err) + } + return acl, nil +} + +func ParseACLOutput(data []byte) (GetBucketAclOutput, error) { + var acl ACL + if err := json.Unmarshal(data, &acl); err != nil { + return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err) + } + + grants := []types.Grant{} + + for _, elem := range acl.Grantees { + acs := elem.Access + grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission}) + } + + return GetBucketAclOutput{ + Owner: &types.Owner{ + ID: &acl.Owner, + }, + AccessControlList: AccessControlList{ + Grants: grants, + }, + }, nil +} + +func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) error { + if acl.Owner != *input.AccessControlPolicy.Owner.ID { + return s3err.GetAPIError(s3err.ErrAccessDenied) + } + + // if the ACL is specified, set the ACL, else replace the grantees + if input.ACL != "" { + acl.ACL = input.ACL + acl.Grantees = []Grantee{} + return nil + } + + grantees := []Grantee{} + + fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{} + + if *input.GrantFullControl != "" { + fullControlList = splitUnique(*input.GrantFullControl, ",") + fmt.Println(fullControlList) + for _, str := range fullControlList { + grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"}) + } + } + if *input.GrantRead != "" { + readList = splitUnique(*input.GrantRead, ",") + for _, str := range readList { + grantees = append(grantees, Grantee{Access: str, Permission: "READ"}) + } + } + if *input.GrantReadACP != "" { + readACPList = splitUnique(*input.GrantReadACP, ",") + for _, str := range readACPList { + grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"}) + } + } + if *input.GrantWrite != "" { + writeList = splitUnique(*input.GrantWrite, ",") + for _, str := range writeList { + grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"}) + } + } + if *input.GrantWriteACP != "" { + writeACPList = splitUnique(*input.GrantWriteACP, ",") + for _, str := range writeACPList { + grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"}) + } + } + + accs := append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...) + + // Check if the specified accounts exist + accList, err := checkIfAccountsExist(accs, iam) + if err != nil { + return err + } + if len(accList) > 0 { + return fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", ")) + } + + acl.Grantees = grantees + acl.ACL = "" + + return nil +} + +func checkIfAccountsExist(accs []string, iam IAMService) ([]string, error) { + result := []string{} + + for _, acc := range accs { + _, err := iam.GetUserAccount(acc) + if err != nil && err != ErrNoSuchUser { + return nil, fmt.Errorf("check user account: %w", err) + } + if err == nil { + result = append(result, acc) + } + } + return result, nil +} + +func splitUnique(s, divider string) []string { + elements := strings.Split(s, divider) + uniqueElements := make(map[string]bool) + result := make([]string, 0, len(elements)) + + for _, element := range elements { + if _, ok := uniqueElements[element]; !ok { + result = append(result, element) + uniqueElements[element] = true + } + } + + return result +} + +func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error { + if isRoot { + return nil + } + + if acl.Owner == access { + return nil + } + + if acl.ACL != "" { + if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") { + return s3err.GetAPIError(s3err.ErrAccessDenied) + } + if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" { + return s3err.GetAPIError(s3err.ErrAccessDenied) + } + + return nil + } else { + grantee := Grantee{Access: access, Permission: permission} + granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"} + + isFound := false + + for _, grt := range acl.Grantees { + if grt == grantee || grt == granteeFullCtrl { + isFound = true + break + } + } + + if isFound { + return nil + } + } + + return s3err.GetAPIError(s3err.ErrAccessDenied) +} + +func IsAdmin(access string, isRoot bool) error { + var data IAMConfig + + if isRoot { + return nil + } + + file, err := os.ReadFile("users.json") + if err != nil { + return fmt.Errorf("unable to read config file: %w", err) + } + + if err := json.Unmarshal(file, &data); err != nil { + return err + } + + acc, ok := data.AccessAccounts[access] + if !ok { + return fmt.Errorf("user does not exist") + } + + if acc.Role == "admin" { + return nil + } + return fmt.Errorf("only admin users have access to this resource") +} diff --git a/auth/iam.go b/auth/iam.go new file mode 100644 index 0000000..14da18c --- /dev/null +++ b/auth/iam.go @@ -0,0 +1,34 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "errors" +) + +// Account is a gateway IAM account +type Account struct { + Secret string `json:"secret"` + Role string `json:"role"` +} + +// IAMService is the interface for all IAM service implementations +type IAMService interface { + CreateAccount(access string, account Account) error + GetUserAccount(access string) (Account, error) + DeleteUserAccount(access string) error +} + +var ErrNoSuchUser = errors.New("user not found") diff --git a/auth/iam_internal.go b/auth/iam_internal.go new file mode 100644 index 0000000..21f2e77 --- /dev/null +++ b/auth/iam_internal.go @@ -0,0 +1,178 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "hash/crc32" + "sync" +) + +// IAMServiceInternal manages the internal IAM service +type IAMServiceInternal struct { + storer Storer + + mu sync.RWMutex + accts IAMConfig + serial uint32 +} + +// UpdateAcctFunc accepts the current data and returns the new data to be stored +type UpdateAcctFunc func([]byte) ([]byte, error) + +// Storer is the interface to manage the peristent IAM data for the internal +// IAM service +type Storer interface { + InitIAM() error + GetIAM() ([]byte, error) + StoreIAM(UpdateAcctFunc) error +} + +// IAMConfig stores all internal IAM accounts +type IAMConfig struct { + AccessAccounts map[string]Account `json:"accessAccounts"` +} + +var _ IAMService = &IAMServiceInternal{} + +// NewInternal creates a new instance for the Internal IAM service +func NewInternal(s Storer) (*IAMServiceInternal, error) { + i := &IAMServiceInternal{ + storer: s, + } + + err := i.updateCache() + if err != nil { + return nil, fmt.Errorf("refresh iam cache: %w", err) + } + + return i, nil +} + +// CreateAccount creates a new IAM account. Returns an error if the account +// already exists. +func (s *IAMServiceInternal) CreateAccount(access string, account Account) error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.storer.StoreIAM(func(data []byte) ([]byte, error) { + var conf IAMConfig + + if len(data) > 0 { + if err := json.Unmarshal(data, &conf); err != nil { + return nil, fmt.Errorf("failed to parse iam: %w", err) + } + } else { + conf.AccessAccounts = make(map[string]Account) + } + + _, ok := conf.AccessAccounts[access] + if ok { + return nil, fmt.Errorf("account already exists") + } + conf.AccessAccounts[access] = account + + b, err := json.Marshal(s.accts) + if err != nil { + return nil, fmt.Errorf("failed to serialize iam: %w", err) + } + + return b, nil + }) +} + +// GetUserAccount retrieves account info for the requested user. Returns +// ErrNoSuchUser if the account does not exist. +func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + data, err := s.storer.GetIAM() + if err != nil { + return Account{}, fmt.Errorf("get iam data: %w", err) + } + + serial := crc32.ChecksumIEEE(data) + if serial != s.serial { + s.mu.RUnlock() + err := s.updateCache() + s.mu.RLock() + if err != nil { + return Account{}, fmt.Errorf("refresh iam cache: %w", err) + } + } + + acct, ok := s.accts.AccessAccounts[access] + if !ok { + return Account{}, ErrNoSuchUser + } + + return acct, nil +} + +// updateCache must be called with no locks held +func (s *IAMServiceInternal) updateCache() error { + s.mu.Lock() + defer s.mu.Unlock() + + data, err := s.storer.GetIAM() + if err != nil { + return fmt.Errorf("get iam data: %w", err) + } + + serial := crc32.ChecksumIEEE(data) + + if len(data) > 0 { + if err := json.Unmarshal(data, &s.accts); err != nil { + return fmt.Errorf("failed to parse the config file: %w", err) + } + } else { + s.accts.AccessAccounts = make(map[string]Account) + } + + s.serial = serial + + return nil +} + +// DeleteUserAccount deletes the specified user account. Does not check if +// account exists. +func (s *IAMServiceInternal) DeleteUserAccount(access string) error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.storer.StoreIAM(func(data []byte) ([]byte, error) { + if len(data) == 0 { + // empty config, do nothing + return data, nil + } + + var conf IAMConfig + + if err := json.Unmarshal(data, &conf); err != nil { + return nil, fmt.Errorf("failed to parse iam: %w", err) + } + + delete(conf.AccessAccounts, access) + + b, err := json.Marshal(s.accts) + if err != nil { + return nil, fmt.Errorf("failed to serialize iam: %w", err) + } + + return b, nil + }) +} diff --git a/backend/auth/acl.go b/backend/auth/acl.go deleted file mode 100644 index 168e3cd..0000000 --- a/backend/auth/acl.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package auth - -import ( - "encoding/json" - "fmt" - "os" - - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/pkg/xattr" - "github.com/versity/versitygw/s3err" -) - -type ACL struct { - ACL types.BucketCannedACL - Owner string - Grantees []Grantee -} - -type Grantee struct { - Permission types.Permission - Access string -} - -type GetBucketAclOutput struct { - Owner *types.Owner - AccessControlList AccessControlList -} - -type AccessControlList struct { - Grants []types.Grant -} - -type ACLService interface { - VerifyACL(bucket, access string, permission types.Permission, isRoot bool) error - IsAdmin(access string, isRoot bool) error -} - -type ACLServiceUnsupported struct{} - -var _ ACLService = &ACLServiceUnsupported{} - -func (ACLServiceUnsupported) VerifyACL(bucket, access string, permission types.Permission, isRoot bool) error { - var ACL ACL - - if isRoot { - return nil - } - - acl, err := xattr.Get(bucket, "user.acl") - if err != nil { - return fmt.Errorf("get acl: %w", err) - } - - if err := json.Unmarshal(acl, &ACL); err != nil { - return fmt.Errorf("parse acl: %w", err) - } - if ACL.Owner == access { - return nil - } - - if ACL.ACL != "" { - if (permission == "READ" || permission == "READ_ACP") && (ACL.ACL != "public-read" && ACL.ACL != "public-read-write") { - return s3err.GetAPIError(s3err.ErrAccessDenied) - } - if (permission == "WRITE" || permission == "WRITE_ACP") && ACL.ACL != "public-read-write" { - return s3err.GetAPIError(s3err.ErrAccessDenied) - } - - return nil - } else { - grantee := Grantee{Access: access, Permission: permission} - granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"} - - isFound := false - - for _, grt := range ACL.Grantees { - if grt == grantee || grt == granteeFullCtrl { - isFound = true - break - } - } - - if isFound { - return nil - } - } - - return s3err.GetAPIError(s3err.ErrAccessDenied) -} - -func (ACLServiceUnsupported) IsAdmin(access string, isRoot bool) error { - var data IAMConfig - - if isRoot { - return nil - } - - file, err := os.ReadFile("users.json") - if err != nil { - return fmt.Errorf("unable to read config file: %w", err) - } - - if err := json.Unmarshal(file, &data); err != nil { - return err - } - - acc, ok := data.AccessAccounts[access] - if !ok { - return fmt.Errorf("user does not exist") - } - - if acc.Role == "admin" { - return nil - } - return fmt.Errorf("only admin users have access to this resource") -} diff --git a/backend/auth/iam.go b/backend/auth/iam.go deleted file mode 100644 index c3b8fbf..0000000 --- a/backend/auth/iam.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package auth - -import ( - "encoding/json" - "fmt" - "os" - "sync" - - "github.com/versity/versitygw/s3err" -) - -type Account struct { - Secret string `json:"secret"` - Role string `json:"role"` - Region string `json:"region"` -} - -type IAMConfig struct { - AccessAccounts map[string]Account `json:"accessAccounts"` -} - -type AccountsCache struct { - mu sync.Mutex - Accounts map[string]Account -} - -func (c *AccountsCache) getAccount(access string) *Account { - c.mu.Lock() - defer c.mu.Unlock() - - acc, ok := c.Accounts[access] - if !ok { - return nil - } - - return &acc -} - -func (c *AccountsCache) updateAccounts() error { - c.mu.Lock() - defer c.mu.Unlock() - - var data IAMConfig - - file, err := os.ReadFile("users.json") - if err != nil { - return fmt.Errorf("error reading config file: %w", err) - } - - if err := json.Unmarshal(file, &data); err != nil { - return fmt.Errorf("error parsing the data: %w", err) - } - - c.Accounts = data.AccessAccounts - - return nil -} - -func (c *AccountsCache) deleteAccount(access string) { - c.mu.Lock() - defer c.mu.Unlock() - delete(c.Accounts, access) -} - -type IAMService interface { - GetIAMConfig() (*IAMConfig, error) - CreateAccount(access string, account *Account) error - GetUserAccount(access string) *Account - DeleteUserAccount(access string) error -} - -type IAMServiceUnsupported struct { - accCache *AccountsCache -} - -var _ IAMService = &IAMServiceUnsupported{} - -func InitIAM() (IAMService, error) { - _, err := os.ReadFile("users.json") - if err != nil { - jsonData, err := json.MarshalIndent(IAMConfig{AccessAccounts: map[string]Account{}}, "", " ") - if err != nil { - return nil, err - } - - if err := os.WriteFile("users.json", jsonData, 0644); err != nil { - return nil, err - } - } - return &IAMServiceUnsupported{accCache: &AccountsCache{Accounts: map[string]Account{}}}, nil -} - -func (IAMServiceUnsupported) GetIAMConfig() (*IAMConfig, error) { - return nil, s3err.GetAPIError(s3err.ErrNotImplemented) -} - -func (s IAMServiceUnsupported) CreateAccount(access string, account *Account) error { - var data IAMConfig - - file, err := os.ReadFile("users.json") - if err != nil { - return fmt.Errorf("unable to read config file: %w", err) - } - - if err := json.Unmarshal(file, &data); err != nil { - return err - } - - _, ok := data.AccessAccounts[access] - if ok { - return fmt.Errorf("user with the given access already exists") - } - - data.AccessAccounts[access] = *account - - updatedJSON, err := json.MarshalIndent(data, "", " ") - if err != nil { - return err - } - - if err := os.WriteFile("users.json", updatedJSON, 0644); err != nil { - return err - } - - return nil -} - -func (s IAMServiceUnsupported) GetUserAccount(access string) *Account { - acc := s.accCache.getAccount(access) - if acc == nil { - err := s.accCache.updateAccounts() - if err != nil { - return nil - } - - return s.accCache.getAccount(access) - } - - return acc -} - -func (s IAMServiceUnsupported) DeleteUserAccount(access string) error { - var data IAMConfig - - file, err := os.ReadFile("users.json") - if err != nil { - return fmt.Errorf("unable to read config file: %w", err) - } - - if err := json.Unmarshal(file, &data); err != nil { - return fmt.Errorf("failed to parse the config file: %w", err) - } - - _, ok := data.AccessAccounts[access] - if !ok { - return fmt.Errorf("invalid access for the user: user does not exist") - } - - delete(data.AccessAccounts, access) - - updatedJSON, err := json.MarshalIndent(data, "", " ") - if err != nil { - return fmt.Errorf("failed to parse the data: %w", err) - } - - if err := os.WriteFile("users.json", updatedJSON, 0644); err != nil { - return fmt.Errorf("failed to saved the changes: %w", err) - } - - s.accCache.deleteAccount(access) - - return nil -} diff --git a/backend/backend.go b/backend/backend.go index 18b8801..18f5342 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -20,7 +20,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -33,9 +32,9 @@ type Backend interface { ListBuckets() (s3response.ListAllMyBucketsResult, error) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) - GetBucketAcl(bucket string) (*auth.GetBucketAclOutput, error) + GetBucketAcl(bucket string) ([]byte, error) PutBucket(bucket, owner string) error - PutBucketAcl(*s3.PutBucketAclInput) error + PutBucketAcl(bucket string, data []byte) error DeleteBucket(bucket string) error CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) @@ -79,7 +78,7 @@ func (BackendUnsupported) String() string { func (BackendUnsupported) ListBuckets() (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutBucketAcl(*s3.PutBucketAclInput) error { +func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error { @@ -91,7 +90,7 @@ func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) GetBucketAcl(bucket string) (*auth.GetBucketAclOutput, error) { +func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { diff --git a/backend/backend_moq_test.go b/backend/backend_moq_test.go index 8a27eb4..d8cc9a4 100644 --- a/backend/backend_moq_test.go +++ b/backend/backend_moq_test.go @@ -6,7 +6,6 @@ package backend import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3response" "io" "sync" @@ -46,7 +45,7 @@ var _ Backend = &BackendMock{} // DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error { // panic("mock out the DeleteObjects method") // }, -// GetBucketAclFunc: func(bucket string) (*auth.GetBucketAclOutput, error) { +// GetBucketAclFunc: func(bucket string) ([]byte, error) { // panic("mock out the GetBucketAcl method") // }, // GetObjectFunc: func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { @@ -85,7 +84,7 @@ var _ Backend = &BackendMock{} // PutBucketFunc: func(bucket string, owner string) error { // panic("mock out the PutBucket method") // }, -// PutBucketAclFunc: func(putBucketAclInput *s3.PutBucketAclInput) error { +// PutBucketAclFunc: func(bucket string, data []byte) error { // panic("mock out the PutBucketAcl method") // }, // PutObjectFunc: func(putObjectInput *s3.PutObjectInput) (string, error) { @@ -147,7 +146,7 @@ type BackendMock struct { DeleteObjectsFunc func(bucket string, objects *s3.DeleteObjectsInput) error // GetBucketAclFunc mocks the GetBucketAcl method. - GetBucketAclFunc func(bucket string) (*auth.GetBucketAclOutput, error) + GetBucketAclFunc func(bucket string) ([]byte, error) // GetObjectFunc mocks the GetObject method. GetObjectFunc func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) @@ -186,7 +185,7 @@ type BackendMock struct { PutBucketFunc func(bucket string, owner string) error // PutBucketAclFunc mocks the PutBucketAcl method. - PutBucketAclFunc func(putBucketAclInput *s3.PutBucketAclInput) error + PutBucketAclFunc func(bucket string, data []byte) error // PutObjectFunc mocks the PutObject method. PutObjectFunc func(putObjectInput *s3.PutObjectInput) (string, error) @@ -390,8 +389,10 @@ type BackendMock struct { } // PutBucketAcl holds details about calls to the PutBucketAcl method. PutBucketAcl []struct { - // PutBucketAclInput is the putBucketAclInput argument value. - PutBucketAclInput *s3.PutBucketAclInput + // Bucket is the bucket argument value. + Bucket string + // Data is the data argument value. + Data []byte } // PutObject holds details about calls to the PutObject method. PutObject []struct { @@ -797,7 +798,7 @@ func (mock *BackendMock) DeleteObjectsCalls() []struct { } // GetBucketAcl calls GetBucketAclFunc. -func (mock *BackendMock) GetBucketAcl(bucket string) (*auth.GetBucketAclOutput, error) { +func (mock *BackendMock) GetBucketAcl(bucket string) ([]byte, error) { if mock.GetBucketAclFunc == nil { panic("BackendMock.GetBucketAclFunc: method is nil but Backend.GetBucketAcl was just called") } @@ -1292,19 +1293,21 @@ func (mock *BackendMock) PutBucketCalls() []struct { } // PutBucketAcl calls PutBucketAclFunc. -func (mock *BackendMock) PutBucketAcl(putBucketAclInput *s3.PutBucketAclInput) error { +func (mock *BackendMock) PutBucketAcl(bucket string, data []byte) error { if mock.PutBucketAclFunc == nil { panic("BackendMock.PutBucketAclFunc: method is nil but Backend.PutBucketAcl was just called") } callInfo := struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte }{ - PutBucketAclInput: putBucketAclInput, + Bucket: bucket, + Data: data, } mock.lockPutBucketAcl.Lock() mock.calls.PutBucketAcl = append(mock.calls.PutBucketAcl, callInfo) mock.lockPutBucketAcl.Unlock() - return mock.PutBucketAclFunc(putBucketAclInput) + return mock.PutBucketAclFunc(bucket, data) } // PutBucketAclCalls gets all the calls that were made to PutBucketAcl. @@ -1312,10 +1315,12 @@ func (mock *BackendMock) PutBucketAcl(putBucketAclInput *s3.PutBucketAclInput) e // // len(mockedBackend.PutBucketAclCalls()) func (mock *BackendMock) PutBucketAclCalls() []struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte } { var calls []struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte } mock.lockPutBucketAcl.RLock() calls = mock.calls.PutBucketAcl diff --git a/backend/backend_test.go b/backend/backend_test.go index fa187dd..f696b74 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -122,7 +121,7 @@ func TestBackend_GetBucketAcl(t *testing.T) { tests = append(tests, test{ name: "get bucket acl error", c: &BackendMock{ - GetBucketAclFunc: func(bucket string) (*auth.GetBucketAclOutput, error) { + GetBucketAclFunc: func(bucket string) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) }, }, diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 35048f0..1834584 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -28,32 +28,50 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" + "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" "github.com/pkg/xattr" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) type Posix struct { + backend.BackendUnsupported + rootfd *os.File rootdir string - backend.BackendUnsupported + + mu sync.RWMutex + iamcache []byte + iamvalid bool + iamexpire time.Time } var _ backend.Backend = &Posix{} +var ( + cacheDuration = 5 * time.Minute +) + const ( metaTmpDir = ".sgwtmp" metaTmpMultipartDir = metaTmpDir + "/multipart" onameAttr = "user.objname" tagHdr = "X-Amz-Tagging" + contentTypeHdr = "content-type" + contentEncHdr = "content-encoding" emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e" + iamFile = "users.json" + iamBackupFile = "users.json.backup" + aclkey = "user.acl" + etagkey = "user.etag" ) func New(rootdir string) (*Posix, error) { @@ -140,7 +158,7 @@ func (p *Posix) PutBucket(bucket string, owner string) error { return fmt.Errorf("marshal acl: %w", err) } - if err := xattr.Set(bucket, "user.acl", jsonACL); err != nil { + if err := xattr.Set(bucket, aclkey, jsonACL); err != nil { return fmt.Errorf("set acl: %w", err) } @@ -263,7 +281,7 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ return nil, s3err.GetAPIError(s3err.ErrInvalidPart) } - b, err := xattr.Get(partPath, "user.etag") + b, err := xattr.Get(partPath, etagkey) etag := string(b) if err != nil { etag = "" @@ -319,7 +337,7 @@ func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts [ // Calculate s3 compatible md5sum for complete multipart. s3MD5 := backend.GetMultipartMD5(parts) - err = xattr.Set(objname, "user.etag", []byte(s3MD5)) + err = xattr.Set(objname, etagkey, []byte(s3MD5)) if err != nil { // cleanup object if returning error os.Remove(objname) @@ -373,22 +391,22 @@ func loadUserMetaData(path string, m map[string]string) (contentType, contentEnc m[strings.TrimPrefix(e, "user.")] = string(b) } - b, err := xattr.Get(path, "user.content-type") + b, err := xattr.Get(path, "user."+contentTypeHdr) contentType = string(b) if err != nil { contentType = "" } if contentType != "" { - m["content-type"] = contentType + m[contentTypeHdr] = contentType } - b, err = xattr.Get(path, "user.content-encoding") + b, err = xattr.Get(path, "user."+contentEncHdr) contentEncoding = string(b) if err != nil { contentEncoding = "" } if contentEncoding != "" { - m["content-encoding"] = contentEncoding + m[contentEncHdr] = contentEncoding } return @@ -626,7 +644,7 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke } partPath := filepath.Join(objdir, uploadID, e.Name()) - b, err := xattr.Get(partPath, "user.etag") + b, err := xattr.Get(partPath, etagkey) etag := string(b) if err != nil { etag = "" @@ -713,7 +731,7 @@ func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum) - xattr.Set(partPath, "user.etag", []byte(etag)) + xattr.Set(partPath, etagkey, []byte(etag)) return etag, nil } @@ -741,7 +759,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { } // set etag attribute to signify this dir was specifically put - xattr.Set(name, "user.etag", []byte(emptyMD5)) + xattr.Set(name, etagkey, []byte(emptyMD5)) return emptyMD5, nil } @@ -779,7 +797,7 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { dataSum := hash.Sum(nil) etag := hex.EncodeToString(dataSum[:]) - xattr.Set(name, "user.etag", []byte(etag)) + xattr.Set(name, etagkey, []byte(etag)) return etag, nil } @@ -816,11 +834,15 @@ func (p *Posix) removeParents(bucket, object string) error { parent := filepath.Dir(objPath) if filepath.Base(parent) == bucket { + // stop removing parents if we hit the bucket directory. break } - _, err := xattr.Get(parent, "user.etag") + _, err := xattr.Get(parent, etagkey) if err == nil { + // a directory with a valid etag means this was specifically + // uploaded with a put object, so stop here and leave this + // directory in place. break } @@ -893,7 +915,7 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer) contentType, contentEncoding := loadUserMetaData(objPath, userMetaData) - b, err := xattr.Get(objPath, "user.etag") + b, err := xattr.Get(objPath, etagkey) etag := string(b) if err != nil { etag = "" @@ -937,7 +959,7 @@ func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) userMetaData := make(map[string]string) contentType, contentEncoding := loadUserMetaData(objPath, userMetaData) - b, err := xattr.Get(objPath, "user.etag") + b, err := xattr.Get(objPath, etagkey) etag := string(b) if err != nil { etag = "" @@ -1010,7 +1032,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( fileSystem := os.DirFS(bucket) results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, func(path string) (bool, error) { - _, err := xattr.Get(filepath.Join(bucket, path), "user.etag") + _, err := xattr.Get(filepath.Join(bucket, path), etagkey) if isNoAttr(err) { return false, nil } @@ -1019,7 +1041,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( } return true, nil }, func(path string) (string, error) { - etag, err := xattr.Get(filepath.Join(bucket, path), "user.etag") + etag, err := xattr.Get(filepath.Join(bucket, path), etagkey) return string(etag), err }, []string{metaTmpDir}) if err != nil { @@ -1051,7 +1073,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) fileSystem := os.DirFS(bucket) results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, func(path string) (bool, error) { - _, err := xattr.Get(filepath.Join(bucket, path), "user.etag") + _, err := xattr.Get(filepath.Join(bucket, path), etagkey) if isNoAttr(err) { return false, nil } @@ -1060,7 +1082,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) } return true, nil }, func(path string) (string, error) { - etag, err := xattr.Get(filepath.Join(bucket, path), "user.etag") + etag, err := xattr.Get(filepath.Join(bucket, path), etagkey) return string(etag), err }, []string{metaTmpDir}) if err != nil { @@ -1080,115 +1102,39 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) }, nil } -func (p *Posix) PutBucketAcl(input *s3.PutBucketAclInput) error { - var ACL auth.ACL - acl, err := xattr.Get(*input.Bucket, "user.acl") +func (p *Posix) PutBucketAcl(bucket string, data []byte) error { + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return s3err.GetAPIError(s3err.ErrNoSuchBucket) + } if err != nil { - return fmt.Errorf("get acl: %w", err) + return fmt.Errorf("stat bucket: %w", err) } - if err := json.Unmarshal(acl, &ACL); err != nil { - return fmt.Errorf("parse acl: %w", err) - } - - if ACL.Owner != *input.AccessControlPolicy.Owner.ID { - return s3err.GetAPIError(s3err.ErrAccessDenied) - } - - // if the ACL is specified, set the ACL, else replace the grantees - if input.ACL != "" { - ACL.ACL = input.ACL - ACL.Grantees = []auth.Grantee{} - } else { - grantees := []auth.Grantee{} - - fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{} - - if *input.GrantFullControl != "" { - fullControlList = splitUnique(*input.GrantFullControl, ",") - fmt.Println(fullControlList) - for _, str := range fullControlList { - grantees = append(grantees, auth.Grantee{Access: str, Permission: "FULL_CONTROL"}) - } - } - if *input.GrantRead != "" { - readList = splitUnique(*input.GrantRead, ",") - for _, str := range readList { - grantees = append(grantees, auth.Grantee{Access: str, Permission: "READ"}) - } - } - if *input.GrantReadACP != "" { - readACPList = splitUnique(*input.GrantReadACP, ",") - for _, str := range readACPList { - grantees = append(grantees, auth.Grantee{Access: str, Permission: "READ_ACP"}) - } - } - if *input.GrantWrite != "" { - writeList = splitUnique(*input.GrantWrite, ",") - for _, str := range writeList { - grantees = append(grantees, auth.Grantee{Access: str, Permission: "WRITE"}) - } - } - if *input.GrantWriteACP != "" { - writeACPList = splitUnique(*input.GrantWriteACP, ",") - for _, str := range writeACPList { - grantees = append(grantees, auth.Grantee{Access: str, Permission: "WRITE_ACP"}) - } - } - - accs := append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...) - - // Check if the specified accounts exist - accList, err := checkIfAccountsExist(accs) - if err != nil { - return err - } - if len(accList) > 0 { - return fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", ")) - } - - ACL.Grantees = grantees - ACL.ACL = "" - } - - ACLJson, err := json.Marshal(ACL) - if err != nil { - return fmt.Errorf("parsing error: %w", err) - } - - if err := xattr.Set(*input.Bucket, "user.acl", ACLJson); err != nil { + if err := xattr.Set(bucket, aclkey, data); err != nil { return fmt.Errorf("set acl: %w", err) } return nil } -func (p *Posix) GetBucketAcl(bucket string) (*auth.GetBucketAclOutput, error) { - var ACL auth.ACL - acl, err := xattr.Get(bucket, "user.acl") +func (p *Posix) GetBucketAcl(bucket string) ([]byte, error) { + _, err := os.Stat(bucket) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + if err != nil { + return nil, fmt.Errorf("stat bucket: %w", err) + } + + b, err := xattr.Get(bucket, aclkey) + if isNoAttr(err) { + return []byte{}, nil + } if err != nil { return nil, fmt.Errorf("get acl: %w", err) } - - if err := json.Unmarshal(acl, &ACL); err != nil { - return nil, fmt.Errorf("parse acl: %w", err) - } - - grants := []types.Grant{} - - for _, elem := range ACL.Grantees { - acs := elem.Access - grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission}) - } - - return &auth.GetBucketAclOutput{ - Owner: &types.Owner{ - ID: &ACL.Owner, - }, - AccessControlList: auth.AccessControlList{ - Grants: grants, - }, - }, nil + return b, nil } func (p *Posix) GetTags(bucket, object string) (map[string]string, error) { @@ -1264,6 +1210,198 @@ func (p *Posix) RemoveTags(bucket, object string) error { return p.SetTags(bucket, object, nil) } +const ( + iamMode = 0600 +) + +func (p *Posix) InitIAM() error { + p.mu.RLock() + defer p.mu.RUnlock() + + _, err := os.ReadFile(iamFile) + if errors.Is(err, fs.ErrNotExist) { + b, err := json.Marshal(auth.IAMConfig{}) + if err != nil { + return fmt.Errorf("marshal default iam: %w", err) + } + err = os.WriteFile(iamFile, b, iamMode) + if err != nil { + return fmt.Errorf("write default iam: %w", err) + } + } + + return nil +} + +func (p *Posix) GetIAM() ([]byte, error) { + p.mu.RLock() + defer p.mu.RUnlock() + + if !p.iamvalid || !p.iamexpire.After(time.Now()) { + p.mu.RUnlock() + err := p.refreshIAM() + p.mu.RLock() + if err != nil { + return nil, err + } + } + + return p.iamcache, nil +} + +const ( + backoff = 100 * time.Millisecond + maxretry = 300 +) + +func (p *Posix) refreshIAM() error { + p.mu.Lock() + defer p.mu.Unlock() + + // We are going to be racing with other running gateways without any + // coordination. So we might find the file does not exist at times. + // For this case we need to retry for a while assuming the other gateway + // will eventually write the file. If it doesn't after the max retries, + // then we will return the error. + + retries := 0 + + for { + b, err := os.ReadFile(iamFile) + if errors.Is(err, fs.ErrNotExist) { + // racing with someone else updating + // keep retrying after backoff + retries++ + if retries < maxretry { + time.Sleep(backoff) + continue + } + return fmt.Errorf("read iam file: %w", err) + } + if err != nil { + return err + } + + p.iamcache = b + p.iamvalid = true + p.iamexpire = time.Now().Add(cacheDuration) + break + } + + return nil +} + +func (p *Posix) StoreIAM(update auth.UpdateAcctFunc) error { + p.mu.Lock() + defer p.mu.Unlock() + + // We are going to be racing with other running gateways without any + // coordination. So the strategy here is to read the current file data. + // If the file doesn't exist, then we assume someone else is currently + // updating the file. So we just need to keep retrying. We also need + // to make sure the data is consistent within a single update. So racing + // writes to a file would possibly leave this in some invalid state. + // We can get atomic updates with rename. If we read the data, update + // the data, write to a temp file, then rename the tempfile back to the + // data file. This should always result in a complete data image. + + // There is at least one unsolved failure mode here. + // If a gateway removes the data file and then crashes, all other + // gateways will retry forever thinking that the original will eventually + // write the file. + + retries := 0 + + for { + b, err := os.ReadFile(iamFile) + if errors.Is(err, fs.ErrNotExist) { + // racing with someone else updating + // keep retrying after backoff + retries++ + if retries < maxretry { + time.Sleep(backoff) + continue + } + + // we have been unsuccessful trying to read the iam file + // so this must be the case where something happened and + // the file did not get updated successfully, and probably + // isn't going to be. The recovery procedure would be to + // copy the backup file into place of the original. + return fmt.Errorf("no iam file, needs backup recovery") + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("read iam file: %w", err) + } + + // reset retries on successful read + retries = 0 + + err = os.Remove(iamFile) + if errors.Is(err, fs.ErrNotExist) { + // racing with someone else updating + // keep retrying after backoff + time.Sleep(backoff) + continue + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("remove old iam file: %w", err) + } + + // save copy of data + datacopy := make([]byte, len(b)) + copy(datacopy, b) + + // make a backup copy in case we crash before update + // this is after remove, so there is a small window something + // can go wrong, but the remove should barrier other gateways + // from trying to write backup at the same time. Only one + // gateway will successfully remove the file. + os.WriteFile(iamBackupFile, b, iamMode) + + b, err = update(b) + if err != nil { + // update failed, try to write old data back out + os.WriteFile(iamFile, datacopy, iamMode) + return fmt.Errorf("update iam data: %w", err) + } + + err = writeTempFile(b) + if err != nil { + // update failed, try to write old data back out + os.WriteFile(iamFile, datacopy, iamMode) + return err + } + + p.iamcache = b + p.iamvalid = true + p.iamexpire = time.Now().Add(cacheDuration) + break + } + + return nil +} + +func writeTempFile(b []byte) error { + f, err := os.CreateTemp(".", iamFile) + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + defer os.Remove(f.Name()) + + _, err = f.Write(b) + if err != nil { + return fmt.Errorf("write temp file: %w", err) + } + + err = os.Rename(f.Name(), iamFile) + if err != nil { + return fmt.Errorf("rename temp file: %w", err) + } + + return nil +} + func isNoAttr(err error) bool { if err == nil { return false @@ -1277,40 +1415,3 @@ func isNoAttr(err error) bool { } return false } - -func checkIfAccountsExist(accs []string) ([]string, error) { - var data auth.IAMConfig - result := []string{} - - file, err := os.ReadFile("users.json") - if err != nil { - return []string{}, fmt.Errorf("unable to read config file: %w", err) - } - - if err := json.Unmarshal(file, &data); err != nil { - return []string{}, err - } - - for _, acc := range accs { - _, ok := data.AccessAccounts[acc] - if !ok { - result = append(result, acc) - } - } - return result, nil -} - -func splitUnique(s, divider string) []string { - elements := strings.Split(s, divider) - uniqueElements := make(map[string]bool) - result := make([]string, 0, len(elements)) - - for _, element := range elements { - if _, ok := uniqueElements[element]; !ok { - result = append(result, element) - uniqueElements[element] = true - } - } - - return result -} diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 1c94504..dd797a9 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -22,8 +22,8 @@ import ( "github.com/gofiber/fiber/v2" "github.com/urfave/cli/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api" "github.com/versity/versitygw/s3api/middlewares" ) @@ -133,7 +133,7 @@ func initFlags() []cli.Flag { } } -func runGateway(be backend.Backend) error { +func runGateway(be backend.Backend, s auth.Storer) error { app := fiber.New(fiber.Config{ AppName: "versitygw", ServerHeader: "VERSITYGW", @@ -161,16 +161,20 @@ func runGateway(be backend.Backend) error { opts = append(opts, s3api.WithDebug()) } - iam, err := auth.InitIAM() + err := s.InitIAM() if err != nil { - return err + return fmt.Errorf("init iam: %w", err) + } + + iam, err := auth.NewInternal(s) + if err != nil { + return fmt.Errorf("setup internal iam service: %w", err) } srv, err := s3api.New(app, be, middlewares.RootUserConfig{ Access: rootUserAccess, Secret: rootUserSecret, - Region: region, - }, port, iam, opts...) + }, port, region, iam, opts...) if err != nil { return fmt.Errorf("init gateway: %v", err) } diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index a7de618..ebd8204 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error { return fmt.Errorf("init posix: %v", err) } - return runGateway(be) + return runGateway(be, be) } diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 1b2d5fa..dd272e9 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -52,5 +52,5 @@ func runScoutfs(ctx *cli.Context) error { return fmt.Errorf("init scoutfs: %v", err) } - return runGateway(be) + return runGateway(be, be) } diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index a9e82dd..2015ab4 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/backend/auth" + "github.com/versity/versitygw/auth" ) type AdminController struct { @@ -26,16 +26,16 @@ type AdminController struct { } func (c AdminController) CreateUser(ctx *fiber.Ctx) error { - access, secret, role, region := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role"), ctx.Query("region") + access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role") requesterRole := ctx.Locals("role") if requesterRole != "admin" { return fmt.Errorf("access denied: only admin users have access to this resource") } - user := auth.Account{Secret: secret, Role: role, Region: region} + user := auth.Account{Secret: secret, Role: role} - err := c.IAMService.CreateAccount(access, &user) + err := c.IAMService.CreateAccount(access, user) if err != nil { return fmt.Errorf("failed to create a user: %w", err) } diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 8f03d22..39adc7a 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -7,7 +7,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3response" "io" "sync" @@ -47,7 +46,7 @@ var _ backend.Backend = &BackendMock{} // DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error { // panic("mock out the DeleteObjects method") // }, -// GetBucketAclFunc: func(bucket string) (*auth.GetBucketAclOutput, error) { +// GetBucketAclFunc: func(bucket string) ([]byte, error) { // panic("mock out the GetBucketAcl method") // }, // GetObjectFunc: func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { @@ -86,7 +85,7 @@ var _ backend.Backend = &BackendMock{} // PutBucketFunc: func(bucket string, owner string) error { // panic("mock out the PutBucket method") // }, -// PutBucketAclFunc: func(putBucketAclInput *s3.PutBucketAclInput) error { +// PutBucketAclFunc: func(bucket string, data []byte) error { // panic("mock out the PutBucketAcl method") // }, // PutObjectFunc: func(putObjectInput *s3.PutObjectInput) (string, error) { @@ -148,7 +147,7 @@ type BackendMock struct { DeleteObjectsFunc func(bucket string, objects *s3.DeleteObjectsInput) error // GetBucketAclFunc mocks the GetBucketAcl method. - GetBucketAclFunc func(bucket string) (*auth.GetBucketAclOutput, error) + GetBucketAclFunc func(bucket string) ([]byte, error) // GetObjectFunc mocks the GetObject method. GetObjectFunc func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) @@ -187,7 +186,7 @@ type BackendMock struct { PutBucketFunc func(bucket string, owner string) error // PutBucketAclFunc mocks the PutBucketAcl method. - PutBucketAclFunc func(putBucketAclInput *s3.PutBucketAclInput) error + PutBucketAclFunc func(bucket string, data []byte) error // PutObjectFunc mocks the PutObject method. PutObjectFunc func(putObjectInput *s3.PutObjectInput) (string, error) @@ -391,8 +390,10 @@ type BackendMock struct { } // PutBucketAcl holds details about calls to the PutBucketAcl method. PutBucketAcl []struct { - // PutBucketAclInput is the putBucketAclInput argument value. - PutBucketAclInput *s3.PutBucketAclInput + // Bucket is the bucket argument value. + Bucket string + // Data is the data argument value. + Data []byte } // PutObject holds details about calls to the PutObject method. PutObject []struct { @@ -798,7 +799,7 @@ func (mock *BackendMock) DeleteObjectsCalls() []struct { } // GetBucketAcl calls GetBucketAclFunc. -func (mock *BackendMock) GetBucketAcl(bucket string) (*auth.GetBucketAclOutput, error) { +func (mock *BackendMock) GetBucketAcl(bucket string) ([]byte, error) { if mock.GetBucketAclFunc == nil { panic("BackendMock.GetBucketAclFunc: method is nil but Backend.GetBucketAcl was just called") } @@ -1293,19 +1294,21 @@ func (mock *BackendMock) PutBucketCalls() []struct { } // PutBucketAcl calls PutBucketAclFunc. -func (mock *BackendMock) PutBucketAcl(putBucketAclInput *s3.PutBucketAclInput) error { +func (mock *BackendMock) PutBucketAcl(bucket string, data []byte) error { if mock.PutBucketAclFunc == nil { panic("BackendMock.PutBucketAclFunc: method is nil but Backend.PutBucketAcl was just called") } callInfo := struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte }{ - PutBucketAclInput: putBucketAclInput, + Bucket: bucket, + Data: data, } mock.lockPutBucketAcl.Lock() mock.calls.PutBucketAcl = append(mock.calls.PutBucketAcl, callInfo) mock.lockPutBucketAcl.Unlock() - return mock.PutBucketAclFunc(putBucketAclInput) + return mock.PutBucketAclFunc(bucket, data) } // PutBucketAclCalls gets all the calls that were made to PutBucketAcl. @@ -1313,10 +1316,12 @@ func (mock *BackendMock) PutBucketAcl(putBucketAclInput *s3.PutBucketAclInput) e // // len(mockedBackend.PutBucketAclCalls()) func (mock *BackendMock) PutBucketAclCalls() []struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte } { var calls []struct { - PutBucketAclInput *s3.PutBucketAclInput + Bucket string + Data []byte } mock.lockPutBucketAcl.RLock() calls = mock.calls.PutBucketAcl diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 7d2df77..2a6b65b 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -29,24 +29,24 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" ) type S3ApiController struct { be backend.Backend - acl auth.ACLService + iam auth.IAMService } func New(be backend.Backend) S3ApiController { - return S3ApiController{be: be, acl: auth.ACLServiceUnsupported{}} + return S3ApiController{be: be} } func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error { access, isRoot := ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) - if err := c.acl.IsAdmin(access, isRoot); err != nil { + if err := auth.IsAdmin(access, isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } res, err := c.be.ListBuckets() @@ -67,6 +67,16 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + if uploadId != "" { if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts)) @@ -75,7 +85,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)) } - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } @@ -84,7 +94,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { } if ctx.Request().URI().QueryArgs().Has("acl") { - if err := c.acl.VerifyACL(bucket, access, "READ_ACP", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } res, err := c.be.GetObjectAcl(bucket, key) @@ -92,14 +102,14 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { } if attrs := ctx.Get("X-Amz-Object-Attributes"); attrs != "" { - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ",")) return SendXMLResponse(ctx, res, err) } - if err := c.acl.VerifyACL(bucket, access, "READ_ACP", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { return SendResponse(ctx, err) } @@ -157,16 +167,27 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { access := ctx.Locals("access").(string) isRoot := ctx.Locals("isRoot").(bool) + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + if ctx.Request().URI().QueryArgs().Has("acl") { - if err := c.acl.VerifyACL(bucket, access, "READ_ACP", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } - res, err := c.be.GetBucketAcl(ctx.Params("bucket")) + + res, err := auth.ParseACLOutput(data) return SendXMLResponse(ctx, res, err) } if ctx.Request().URI().QueryArgs().Has("uploads") { - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } res, err := c.be.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))}) @@ -174,14 +195,14 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { } if ctx.QueryInt("list-type") == 2 { - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys) return SendXMLResponse(ctx, res, err) } - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } @@ -212,11 +233,21 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { return errors.New("wrong api call") } - if err := c.acl.VerifyACL(bucket, access, "WRITE_ACP", isRoot); err != nil { + data, err := c.be.GetBucketAcl(bucket) + if err != nil { return SendResponse(ctx, err) } - err := c.be.PutBucketAcl(&s3.PutBucketAclInput{ + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil { + return SendResponse(ctx, err) + } + + input := &s3.PutBucketAclInput{ Bucket: &bucket, ACL: types.BucketCannedACL(acl), GrantFullControl: &grantFullControl, @@ -225,8 +256,9 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { GrantWrite: &granWrite, GrantWriteACP: &grantWriteACP, AccessControlPolicy: &types.AccessControlPolicy{Owner: &types.Owner{ID: &access}}, - }) + } + err = auth.UpdateACL(input, parsedAcl, c.iam) return SendResponse(ctx, err) } @@ -280,13 +312,23 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } } + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + if uploadId != "" && partNumberStr != "" { partNumber := ctx.QueryInt("partNumber", -1) if partNumber < 1 { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart)) } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendResponse(ctx, err) } @@ -302,7 +344,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { return errors.New("wrong api call") } - if err := c.acl.VerifyACL(bucket, access, "WRITE_ACP", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE_ACP", isRoot); err != nil { return SendResponse(ctx, err) } @@ -325,7 +367,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { copySourceSplit := strings.Split(copySource, "/") srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:] - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } @@ -335,7 +377,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { metadata := utils.GetUserMetaData(&ctx.Request().Header) - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendResponse(ctx, err) } @@ -352,10 +394,22 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + + data, err := c.be.GetBucketAcl(bucket) + if err != nil { return SendResponse(ctx, err) } - err := c.be.DeleteBucket(bucket) + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { + return SendResponse(ctx, err) + } + + err = c.be.DeleteBucket(bucket) return SendResponse(ctx, err) } @@ -367,11 +421,21 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { return errors.New("wrong api call") } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + data, err := c.be.GetBucketAcl(bucket) + if err != nil { return SendResponse(ctx, err) } - err := c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj}) + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { + return SendResponse(ctx, err) + } + + err = c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj}) return SendResponse(ctx, err) } @@ -387,10 +451,20 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + if uploadId != "" { expectedBucketOwner, requestPayer := ctx.Get("X-Amz-Expected-Bucket-Owner"), ctx.Get("X-Amz-Request-Payer") - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendResponse(ctx, err) } @@ -404,21 +478,32 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { return SendResponse(ctx, err) } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendResponse(ctx, err) } - err := c.be.DeleteObject(bucket, key) + err = c.be.DeleteObject(bucket, key) return SendResponse(ctx, err) } func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + + data, err := c.be.GetBucketAcl(bucket) + if err != nil { return SendResponse(ctx, err) } - _, err := c.be.HeadBucket(bucket) + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { + return SendResponse(ctx, err) + } + + _, err = c.be.HeadBucket(bucket) // TODO: set bucket response headers return SendResponse(ctx, err) } @@ -435,7 +520,17 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - if err := c.acl.VerifyACL(bucket, access, "READ", isRoot); err != nil { + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + + if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendResponse(ctx, err) } @@ -490,6 +585,16 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } + data, err := c.be.GetBucketAcl(bucket) + if err != nil { + return SendResponse(ctx, err) + } + + parsedAcl, err := auth.ParseACL(data) + if err != nil { + return SendResponse(ctx, err) + } + var restoreRequest s3.RestoreObjectInput if ctx.Request().URI().QueryArgs().Has("restore") { xmlErr := xml.Unmarshal(ctx.Body(), &restoreRequest) @@ -497,7 +602,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { return errors.New("wrong api call") } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendResponse(ctx, err) } @@ -514,7 +619,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { return errors.New("wrong api call") } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } @@ -522,7 +627,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { return SendXMLResponse(ctx, res, err) } - if err := c.acl.VerifyACL(bucket, access, "WRITE", isRoot); err != nil { + if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendXMLResponse(ctx, nil, err) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index abf387a..f52e89f 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -15,6 +15,7 @@ package controllers import ( + "encoding/json" "io" "net/http" "net/http/httptest" @@ -26,19 +27,31 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) +var ( + acl auth.ACL + acldata []byte +) + +func init() { + var err error + acldata, err = json.Marshal(acl) + if err != nil { + panic(err) + } +} + func TestNew(t *testing.T) { type args struct { be backend.Backend } be := backend.BackendUnsupported{} - acl := auth.ACLServiceUnsupported{} tests := []struct { name string @@ -51,8 +64,7 @@ func TestNew(t *testing.T) { be: be, }, want: S3ApiController{ - be: be, - acl: acl, + be: be, }, }, } @@ -73,11 +85,13 @@ func TestS3ApiController_ListBuckets(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -91,11 +105,13 @@ func TestS3ApiController_ListBuckets(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) }, }, - acl: auth.ACLServiceUnsupported{}, } appErr.Use(func(ctx *fiber.Ctx) error { @@ -154,6 +170,9 @@ func TestS3ApiController_GetActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { return s3response.ListPartsResponse{}, nil }, @@ -167,7 +186,6 @@ func TestS3ApiController_GetActions(t *testing.T) { return &s3.GetObjectOutput{Metadata: nil}, nil }, }, - acl: &auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { ctx.Locals("access", "valid access") @@ -264,8 +282,8 @@ func TestS3ApiController_ListActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) (*auth.GetBucketAclOutput, error) { - return nil, nil + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil }, ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { return s3response.ListMultipartUploadsResponse{}, nil @@ -277,7 +295,6 @@ func TestS3ApiController_ListActions(t *testing.T) { return &s3.ListObjectsOutput{}, nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -291,11 +308,13 @@ func TestS3ApiController_ListActions(t *testing.T) { //Error case s3ApiControllerError := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) }, }, - acl: auth.ACLServiceUnsupported{}, } appError := fiber.New() appError.Use(func(ctx *fiber.Ctx) error { @@ -381,14 +400,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - PutBucketAclFunc: func(*s3.PutBucketAclInput) error { + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, + PutBucketAclFunc: func(string, []byte) error { return nil }, PutBucketFunc: func(bucket, owner string) error { return nil }, }, - acl: auth.ACLServiceUnsupported{}, } // Mock ctx.Locals app.Use(func(ctx *fiber.Ctx) error { @@ -463,6 +484,9 @@ func TestS3ApiController_PutActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, UploadPartCopyFunc: func(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { return &s3.UploadPartCopyOutput{}, nil }, @@ -476,7 +500,6 @@ func TestS3ApiController_PutActions(t *testing.T) { return "Hey", nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { ctx.Locals("access", "valid access") @@ -601,11 +624,13 @@ func TestS3ApiController_DeleteBucket(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, DeleteBucketFunc: func(bucket string) error { return nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -621,11 +646,13 @@ func TestS3ApiController_DeleteBucket(t *testing.T) { s3ApiControllerErr := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, DeleteBucketFunc: func(bucket string) error { return s3err.GetAPIError(48) }, }, - acl: auth.ACLServiceUnsupported{}, } appErr.Use(func(ctx *fiber.Ctx) error { @@ -682,11 +709,13 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error { return nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -749,6 +778,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, DeleteObjectFunc: func(bucket, object string) error { return nil }, @@ -756,7 +788,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) { return nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -770,6 +801,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, DeleteObjectFunc: func(bucket, object string) error { return s3err.GetAPIError(7) }, @@ -838,11 +872,13 @@ func TestS3ApiController_HeadBucket(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { return &s3.HeadBucketOutput{}, nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -857,11 +893,13 @@ func TestS3ApiController_HeadBucket(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { return nil, s3err.GetAPIError(3) }, }, - acl: auth.ACLServiceUnsupported{}, } appErr.Use(func(ctx *fiber.Ctx) error { @@ -926,6 +964,9 @@ func TestS3ApiController_HeadObject(t *testing.T) { s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{ ContentEncoding: &contentEncoding, @@ -936,7 +977,6 @@ func TestS3ApiController_HeadObject(t *testing.T) { }, nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { @@ -951,11 +991,13 @@ func TestS3ApiController_HeadObject(t *testing.T) { s3ApiControllerErr := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(42) }, }, - acl: auth.ACLServiceUnsupported{}, } appErr.Use(func(ctx *fiber.Ctx) error { @@ -1011,6 +1053,9 @@ func TestS3ApiController_CreateActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ + GetBucketAclFunc: func(bucket string) ([]byte, error) { + return acldata, nil + }, RestoreObjectFunc: func(bucket, object string, restoreRequest *s3.RestoreObjectInput) error { return nil }, @@ -1021,7 +1066,6 @@ func TestS3ApiController_CreateActions(t *testing.T) { return &s3.CreateMultipartUploadOutput{}, nil }, }, - acl: auth.ACLServiceUnsupported{}, } app.Use(func(ctx *fiber.Ctx) error { diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 8858922..5c42be1 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -25,7 +25,7 @@ import ( v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/smithy-go/logging" "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/backend/auth" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -38,10 +38,9 @@ const ( type RootUserConfig struct { Access string Secret string - Region string } -func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, debug bool) fiber.Handler { +func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler { acct := accounts{root: root, iam: iam} return func(ctx *fiber.Ctx) error { @@ -74,10 +73,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, debug bool) fib } signedHdrs := strings.Split(signHdrKv[1], ";") - account := acct.getAccount(creds[0]) - if account == nil { + account, err := acct.getAccount(creds[0]) + if err == auth.ErrNoSuchUser { return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)) } + if err != nil { + return controllers.SendResponse(ctx, err) + } // Check X-Amz-Date header date := ctx.Get("X-Amz-Date") @@ -113,7 +115,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, debug bool) fib signErr := signer.SignHTTP(req.Context(), aws.Credentials{ AccessKeyID: creds[0], SecretAccessKey: account.Secret, - }, req, hexPayload, creds[3], account.Region, tdate, func(options *v4.SignerOptions) { + }, req, hexPayload, creds[3], region, tdate, func(options *v4.SignerOptions) { if debug { options.LogSigning = true options.Logger = logging.NewStandardLogger(os.Stderr) @@ -147,16 +149,13 @@ type accounts struct { iam auth.IAMService } -func (a accounts) getAccount(access string) *auth.Account { - var account *auth.Account +func (a accounts) getAccount(access string) (auth.Account, error) { if access == a.root.Access { - account = &auth.Account{ + return auth.Account{ Secret: a.root.Secret, Role: "admin", - Region: a.root.Region, - } - } else { - account = a.iam.GetUserAccount(access) + }, nil } - return account + + return a.iam.GetUserAccount(access) } diff --git a/s3api/router.go b/s3api/router.go index 13d881f..8e753f1 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -16,8 +16,8 @@ package s3api import ( "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api/controllers" ) diff --git a/s3api/router_test.go b/s3api/router_test.go index 2b14dd9..5d80471 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -18,8 +18,8 @@ import ( "testing" "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" ) func TestS3ApiRouter_Init(t *testing.T) { @@ -39,7 +39,7 @@ func TestS3ApiRouter_Init(t *testing.T) { args: args{ app: fiber.New(), be: backend.BackendUnsupported{}, - iam: auth.IAMServiceUnsupported{}, + iam: &auth.IAMServiceInternal{}, }, }, } diff --git a/s3api/server.go b/s3api/server.go index 0f33db2..612e915 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -19,8 +19,8 @@ import ( "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/logger" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api/middlewares" ) @@ -33,7 +33,7 @@ type S3ApiServer struct { debug bool } -func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) { +func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...Option) (*S3ApiServer, error) { server := &S3ApiServer{ app: app, backend: be, @@ -50,7 +50,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po app.Use(middlewares.RequestLogger) // Authentication middlewares - app.Use(middlewares.VerifyV4Signature(root, iam, server.debug)) + app.Use(middlewares.VerifyV4Signature(root, iam, region, server.debug)) app.Use(middlewares.VerifyMD5Body()) server.router.Init(app, be, iam) diff --git a/s3api/server_test.go b/s3api/server_test.go index ea6aa1c..a5bf975 100644 --- a/s3api/server_test.go +++ b/s3api/server_test.go @@ -19,8 +19,8 @@ import ( "testing" "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/auth" "github.com/versity/versitygw/s3api/middlewares" ) @@ -63,7 +63,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root, - tt.args.port, auth.IAMServiceUnsupported{}) + tt.args.port, "us-east-1", &auth.IAMServiceInternal{}) if (err != nil) != tt.wantErr { t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) return