Compare commits

...

4 Commits

Author SHA1 Message Date
niksis02
668cad8fe9 fix: moves the body stream reader to public buckets middleware 2025-07-01 23:58:00 +04:00
niksis02
64e705f49a feat: adds the bucket/object name validator middleware 2025-07-01 23:51:49 +04:00
niksis02
96c4c3e2d6 feat: implements public bucket access.
This implementation introduces **public buckets**, which are accessible without signature-based authentication.

There are two ways to grant public access to a bucket:

* **Bucket ACLs**
* **Bucket Policies**

Only `Get` and `List` operations are permitted on public buckets. All **write operations** require authentication, regardless of whether public access is granted through an ACL or a policy.

The implementation includes an `AuthorizePublicBucketAccess` middleware, which checks if public access has been granted to the bucket. If so, authentication middlewares are skipped. For unauthenticated requests, appropriate errors are returned based on the specific S3 action.

---

**1. Bucket-Level Operations:**

```json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": "arn:aws:s3:::test"
    }
  ]
}
```

**2. Object-Level Operations:**

```json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": "arn:aws:s3:::test/*"
    }
  ]
}
```

**3. Both Bucket and Object-Level Operations:**

```json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": "arn:aws:s3:::test"
    },
    {
      "Effect": "Allow",
      "Principal": "*",
      "Action": "s3:*",
      "Resource": "arn:aws:s3:::test/*"
    }
  ]
}
```

---

```sh
aws s3api create-bucket --bucket test --object-ownership BucketOwnerPreferred
aws s3api put-bucket-acl --bucket test --acl public-read
```
2025-07-01 10:26:47 -07:00
Ben McClelland
74dceb849e fix: validate object names before sending to backend 2025-07-01 10:26:40 -07:00
29 changed files with 3881 additions and 632 deletions

201
auth/access-control.go Normal file
View File

@@ -0,0 +1,201 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"errors"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
// Verify destination bucket access
if err := VerifyAccess(ctx, be, opts); err != nil {
return err
}
// Verify source bucket access
srcBucket, srcObject, found := strings.Cut(copySource, "/")
if !found {
return s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
// Get source bucket ACL
srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
if err != nil {
return err
}
var srcBucketAcl ACL
if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
return err
}
if err := VerifyAccess(ctx, be, AccessOptions{
Acl: srcBucketAcl,
AclPermission: PermissionRead,
IsRoot: opts.IsRoot,
Acc: opts.Acc,
Bucket: srcBucket,
Object: srcObject,
Action: GetObjectAction,
}); err != nil {
return err
}
return nil
}
type AccessOptions struct {
Acl ACL
AclPermission Permission
IsRoot bool
Acc Account
Bucket string
Object string
Action Action
Readonly bool
IsBucketPublic bool
}
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
// Skip the access check for public buckets
if opts.IsBucketPublic {
return nil
}
if opts.Readonly {
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
if policyErr != nil {
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return policyErr
}
} else {
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
}
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
return err
}
return nil
}
// Detects if the action is policy related
// e.g.
// 'GetBucketPolicy', 'PutBucketPolicy'
func isPolicyAction(action Action) bool {
return action == GetBucketPolicyAction || action == PutBucketPolicyAction
}
// VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy
func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error {
// ACL disabled
policy, err := be.GetBucketPolicy(ctx, bucket)
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return err
}
if err == nil {
err = VerifyPublicBucketPolicy(policy, bucket, object, action)
if err == nil {
// if ACLs are disabled, and the bucket grants public access,
// policy actions should return 'MethodNotAllowed'
if isPolicyAction(action) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil
}
}
// if the action is not in the ACL whitelist the access is denied
_, ok := publicACLAllowedActions[action]
if !ok {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
err = VerifyPublicBucketACL(ctx, be, bucket, action, permission)
if err != nil {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
func MayCreateBucket(acct Account, isRoot bool) error {
if isRoot {
return nil
}
if acct.Role == RoleUser {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
// Owner check
if acct.Access == acl.Owner {
return nil
}
// Root user has access over almost everything
if isRoot {
return nil
}
// Admin user case
if acct.Role == RoleAdmin {
return nil
}
// Return access denied in all other cases
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
type PublicACLAllowedActions map[Action]struct{}
var publicACLAllowedActions PublicACLAllowedActions = PublicACLAllowedActions{
ListBucketAction: struct{}{},
PutObjectAction: struct{}{},
ListBucketMultipartUploadsAction: struct{}{},
DeleteObjectAction: struct{}{},
ListBucketVersionsAction: struct{}{},
GetObjectAction: struct{}{},
GetObjectAttributesAction: struct{}{},
GetObjectAclAction: struct{}{},
}

View File

@@ -33,6 +33,17 @@ type ACL struct {
Grantees []Grantee
}
// IsPublic specifies if the acl grants public read access
func (acl *ACL) IsPublic(permission Permission) bool {
for _, grt := range acl.Grantees {
if grt.Permission == permission && grt.Type == types.TypeGroup && grt.Access == "all-users" {
return true
}
}
return false
}
type Grantee struct {
Permission Permission
Access string
@@ -435,117 +446,22 @@ func verifyACL(acl ACL, access string, permission Permission) error {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
func MayCreateBucket(acct Account, isRoot bool) error {
if isRoot {
return nil
}
if acct.Role == RoleUser {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
// Owner check
if acct.Access == acl.Owner {
return nil
}
// Root user has access over almost everything
if isRoot {
return nil
}
// Admin user case
if acct.Role == RoleAdmin {
return nil
}
// Return access denied in all other cases
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
type AccessOptions struct {
Acl ACL
AclPermission Permission
IsRoot bool
Acc Account
Bucket string
Object string
Action Action
Readonly bool
}
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
if opts.Readonly {
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
if policyErr != nil {
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return policyErr
}
} else {
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
}
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
return err
}
return nil
}
func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
// Verify destination bucket access
if err := VerifyAccess(ctx, be, opts); err != nil {
return err
}
// Verify source bucket access
srcBucket, srcObject, found := strings.Cut(copySource, "/")
if !found {
return s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
// Get source bucket ACL
srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
// Verifies if the bucket acl grants public access
func VerifyPublicBucketACL(ctx context.Context, be backend.Backend, bucket string, action Action, permission Permission) error {
aclBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{
Bucket: &bucket,
})
if err != nil {
return err
}
var srcBucketAcl ACL
if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
acl, err := ParseACL(aclBytes)
if err != nil {
return err
}
if err := VerifyAccess(ctx, be, AccessOptions{
Acl: srcBucketAcl,
AclPermission: PermissionRead,
IsRoot: opts.IsRoot,
Acc: opts.Acc,
Bucket: srcBucket,
Object: srcObject,
Action: GetObjectAction,
}); err != nil {
return err
if !acl.IsPublic(permission) {
return ErrAccessDenied
}
return nil

View File

@@ -22,6 +22,8 @@ import (
"github.com/versity/versitygw/s3err"
)
var ErrAccessDenied = errors.New("access denied")
type policyErr string
func (p policyErr) Error() string {
@@ -89,6 +91,24 @@ func (bp *BucketPolicy) isAllowed(principal string, action Action, resource stri
return isAllowed
}
// isPublic checks if the bucket policy statements contain
// an entity granting public access
func (bp *BucketPolicy) isPublic(resource string, action Action) bool {
var isAllowed bool
for _, statement := range bp.Statement {
if statement.isPublic(resource, action) {
switch statement.Effect {
case BucketPolicyAccessTypeAllow:
isAllowed = true
case BucketPolicyAccessTypeDeny:
return false
}
}
}
return isAllowed
}
type BucketPolicyItem struct {
Effect BucketPolicyAccessType `json:"Effect"`
Principals Principals `json:"Principal"`
@@ -134,6 +154,11 @@ func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource
return false
}
// isPublic checks if the bucket policy statemant grants public access
func (bpi *BucketPolicyItem) isPublic(resource string, action Action) bool {
return bpi.Principals.IsPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource)
}
func getMalformedPolicyError(err error) error {
return s3err.APIError{
Code: "MalformedPolicy",
@@ -183,3 +208,22 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
return nil
}
// Checks if the bucket policy grants public access
func VerifyPublicBucketPolicy(policy []byte, bucket, object string, action Action) error {
var bucketPolicy BucketPolicy
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
return err
}
resource := bucket
if object != "" {
resource += "/" + object
}
if !bucketPolicy.isPublic(resource, action) {
return ErrAccessDenied
}
return nil
}

View File

@@ -91,6 +91,7 @@ var supportedActionList = map[Action]struct{}{
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
GetBucketObjectLockConfigurationAction: {},
PutBucketObjectLockConfigurationAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},

View File

@@ -121,3 +121,10 @@ func (p Principals) Contains(userAccess string) bool {
_, found := p[userAccess]
return found
}
// Bucket policy grants public access, if it contains
// a wildcard match to all the users
func (p Principals) IsPublic() bool {
_, ok := p["*"]
return ok
}

View File

@@ -136,7 +136,7 @@ func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResu
}
}
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend) error {
data, err := be.GetObjectLockConfiguration(ctx, bucket)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
@@ -211,7 +211,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
if isBucketPublic {
err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
} else {
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
}
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
@@ -254,7 +258,11 @@ func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects [
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
if isBucketPublic {
err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
} else {
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
}
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}

File diff suppressed because it is too large Load Diff

View File

@@ -32,6 +32,7 @@ import (
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
@@ -99,8 +100,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
ctx.Locals("isDebug", false)
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
return ctx.Next()
})
app.Get("/", s3ApiController.ListBuckets)
@@ -116,8 +116,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
}
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access", Role: "admin:"})
ctx.Locals("isDebug", false)
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"})
return ctx.Next()
})
appErr.Get("/", s3ApiControllerErr.ListBuckets)
@@ -220,10 +219,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
},
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Get("/:bucket/:key/*", s3ApiController.GetActions)
@@ -413,10 +411,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
@@ -438,10 +435,9 @@ func TestS3ApiController_ListActions(t *testing.T) {
}
appError := fiber.New()
appError.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
appError.Get("/:bucket", s3ApiControllerError.ListActions)
@@ -707,10 +703,9 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
}
// Mock ctx.Locals
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{Owner: "valid access"})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{Owner: "valid access"})
return ctx.Next()
})
app.Put("/:bucket", s3ApiController.PutBucketActions)
@@ -1003,10 +998,9 @@ func TestS3ApiController_PutActions(t *testing.T) {
},
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Put("/:bucket/:key/*", s3ApiController.PutActions)
@@ -1292,10 +1286,9 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
@@ -1378,10 +1371,9 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Post("/:bucket", s3ApiController.DeleteObjects)
@@ -1458,10 +1450,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
@@ -1482,10 +1473,9 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
}}
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions)
@@ -1565,11 +1555,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
utils.ContextKeyRegion.Set(ctx, "us-east-1")
return ctx.Next()
})
@@ -1583,17 +1572,16 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
return acldata, nil
},
HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(3)
return nil, s3err.GetAPIError(s3err.ErrBucketNotEmpty)
},
},
}
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
utils.ContextKeyRegion.Set(ctx, "us-east-1")
return ctx.Next()
})
@@ -1670,10 +1658,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
}
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
@@ -1693,10 +1680,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
}
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
@@ -1798,10 +1784,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
`
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)

View File

@@ -24,6 +24,7 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -34,7 +35,6 @@ var (
func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
path := ctx.Path()
pathParts := strings.Split(path, "/")
bucket := pathParts[1]
@@ -53,6 +53,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
!ctx.Request().URI().QueryArgs().Has("object-lock") &&
!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
!ctx.Request().URI().QueryArgs().Has("cors") {
isRoot, acct := utils.ContextKeyIsRoot.Get(ctx).(bool), utils.ContextKeyAccount.Get(ctx).(auth.Account)
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
@@ -77,10 +78,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
// if owner is not set, set default owner to root account
if parsedAcl.Owner == "" {
parsedAcl.Owner = ctx.Locals("rootAccess").(string)
parsedAcl.Owner = utils.ContextKeyRootAccessKey.Get(ctx).(string)
}
ctx.Locals("parsedAcl", parsedAcl)
utils.ContextKeyParsedAcl.Set(ctx, parsedAcl)
return ctx.Next()
}
}

View File

@@ -21,13 +21,14 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
if acct.Role != auth.RoleAdmin {
path := ctx.Path()
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),

View File

@@ -46,14 +46,15 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
// If account is set in context locals, it means it was presigned url case
_, ok := ctx.Locals("account").(auth.Account)
if ok {
// The bucket is public, no need to check this signature
if utils.ContextKeyPublicBucket.IsSet(ctx) {
return ctx.Next()
}
// If ContextKeyAuthenticated is set in context locals, it means it was presigned url case
if utils.ContextKeyAuthenticated.IsSet(ctx) {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
@@ -72,8 +73,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
}, logger, mm)
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("rootAccess", root.Access)
utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
@@ -82,7 +82,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
if err != nil {
return sendResponse(ctx, err, logger, mm)
}
ctx.Locals("account", account)
utils.ContextKeyAccount.Set(ctx, account)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")

View File

@@ -18,14 +18,15 @@ import (
"io"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/utils"
)
func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
r, ok := ctx.Locals("body-reader").(io.Reader)
r, ok := utils.ContextKeyBodyReader.Get(ctx).(io.Reader)
if !ok {
r = ctx.Request().BodyStream()
}
r = wr(r)
ctx.Locals("body-reader", r)
utils.ContextKeyBodyReader.Set(ctx, r)
}

View File

@@ -0,0 +1,36 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func ValidateBucketObjectNames(l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
return func(ctx *fiber.Ctx) error {
bucket, object := parsePath(ctx.Path())
if bucket != "" && !utils.IsValidBucketName(bucket) {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), l, mm)
}
if object != "" && !utils.IsObjectNameValid(object) {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrBadRequest), l, mm)
}
return ctx.Next()
}
}

View File

@@ -17,7 +17,6 @@ package middlewares
import (
"io"
"strconv"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
@@ -31,20 +30,24 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
// The bucket is public, no need to check this signature
if utils.ContextKeyPublicBucket.IsSet(ctx) {
return ctx.Next()
}
if ctx.Query("X-Amz-Signature") == "" {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
// Set in the context the "authenticated" key, in case the authentication succeeds,
// otherwise the middleware will return the caucht error
utils.ContextKeyAuthenticated.Set(ctx, true)
authData, err := utils.ParsePresignedURIParts(ctx)
if err != nil {
return sendResponse(ctx, err, logger, mm)
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("rootAccess", root.Access)
utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
@@ -53,7 +56,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
if err != nil {
return sendResponse(ctx, err, logger, mm)
}
ctx.Locals("account", account)
utils.ContextKeyAccount.Set(ctx, account)
var contentLength int64
contentLengthStr := ctx.Get("Content-Length")

View File

@@ -0,0 +1,298 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
return func(ctx *fiber.Ctx) error {
// skip for auhtneicated requests
if ctx.Query("X-Amz-Algorithm") != "" || ctx.Get("Authorization") != "" {
return ctx.Next()
}
bucket, object := parsePath(ctx.Path())
action, permission, err := detectS3Action(ctx, object == "")
if err != nil {
return sendResponse(ctx, err, l, mm)
}
err = auth.VerifyPublicAccess(ctx.Context(), be, action, permission, bucket, object)
if err != nil {
return sendResponse(ctx, err, l, mm)
}
if utils.IsBigDataAction(ctx) {
payloadType := ctx.Get("X-Amz-Content-Sha256")
if utils.IsUnsignedStreamingPayload(payloadType) {
checksumType, err := utils.ExtractChecksumType(ctx)
if err != nil {
return sendResponse(ctx, err, l, mm)
}
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr io.Reader
cr, err = utils.NewUnsignedChunkReader(r, checksumType)
return cr
})
if err != nil {
return sendResponse(ctx, err, l, mm)
}
}
}
utils.ContextKeyPublicBucket.Set(ctx, true)
return ctx.Next()
}
}
func detectS3Action(ctx *fiber.Ctx, isBucketAction bool) (auth.Action, auth.Permission, error) {
path := ctx.Path()
// ListBuckets is not publically available
if path == "/" {
//TODO: Still not clear what kind of error should be returned in this case(ListBuckets)
return "", auth.PermissionRead, s3err.GetAPIError(s3err.ErrAccessDenied)
}
queryArgs := ctx.Context().QueryArgs()
switch ctx.Method() {
case fiber.MethodPatch:
// Admin apis should always be protected
return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
case fiber.MethodHead:
// HeadBucket
if isBucketAction {
return auth.ListBucketAction, auth.PermissionRead, nil
}
// HeadObject
return auth.GetObjectAction, auth.PermissionRead, nil
case fiber.MethodGet:
if isBucketAction {
if queryArgs.Has("tagging") {
// GetBucketTagging
return auth.GetBucketTaggingAction, auth.PermissionRead, nil
} else if queryArgs.Has("ownershipControls") {
// GetBucketOwnershipControls
return auth.GetBucketOwnershipControlsAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousGetBucketOwnership)
} else if queryArgs.Has("versioning") {
// GetBucketVersioning
return auth.GetBucketVersioningAction, auth.PermissionRead, nil
} else if queryArgs.Has("policy") {
// GetBucketPolicy
return auth.GetBucketPolicyAction, auth.PermissionRead, nil
} else if queryArgs.Has("cors") {
// GetBucketCors
return auth.GetBucketCorsAction, auth.PermissionRead, nil
} else if queryArgs.Has("versions") {
// ListObjectVersions
return auth.ListBucketVersionsAction, auth.PermissionRead, nil
} else if queryArgs.Has("object-lock") {
// GetObjectLockConfiguration
return auth.GetBucketObjectLockConfigurationAction, auth.PermissionReadAcp, nil
} else if queryArgs.Has("acl") {
// GetBucketAcl
return auth.GetBucketAclAction, auth.PermissionRead, nil
} else if queryArgs.Has("uploads") {
// ListMultipartUploads
return auth.ListBucketMultipartUploadsAction, auth.PermissionRead, nil
} else if queryArgs.GetUintOrZero("list-type") == 2 {
// ListObjectsV2
return auth.ListBucketAction, auth.PermissionRead, nil
}
// All the other requests are considerd as ListObjects in the router
// no matter what kind of query arguments are provided apart from the ones above
return auth.ListBucketAction, auth.PermissionRead, nil
}
if queryArgs.Has("tagging") {
// GetObjectTagging
return auth.GetObjectTaggingAction, auth.PermissionRead, nil
} else if queryArgs.Has("retention") {
// GetObjectRetention
return auth.GetObjectRetentionAction, auth.PermissionRead, nil
} else if queryArgs.Has("legal-hold") {
// GetObjectLegalHold
return auth.GetObjectLegalHoldAction, auth.PermissionReadAcp, nil
} else if queryArgs.Has("acl") {
// GetObjectAcl
return auth.GetObjectAclAction, auth.PermissionRead, nil
} else if queryArgs.Has("attributes") {
// GetObjectAttributes
return auth.GetObjectAttributesAction, auth.PermissionRead, nil
} else if queryArgs.Has("uploadId") {
// ListParts
return auth.ListMultipartUploadPartsAction, auth.PermissionRead, nil
}
// All the other requests are considerd as GetObject in the router
// no matter what kind of query arguments are provided apart from the ones above
if queryArgs.Has("versionId") {
return auth.GetObjectVersionAction, auth.PermissionRead, nil
}
return auth.GetObjectAction, auth.PermissionRead, nil
case fiber.MethodPut:
if isBucketAction {
if queryArgs.Has("tagging") {
// PutBucketTagging
return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
}
if queryArgs.Has("ownershipControls") {
// PutBucketOwnershipControls
return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
}
if queryArgs.Has("versioning") {
// PutBucketVersioning
return auth.PutBucketVersioningAction, auth.PermissionWrite, nil
}
if queryArgs.Has("object-lock") {
// PutObjectLockConfiguration
return auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite, nil
}
if queryArgs.Has("cors") {
// PutBucketCors
return auth.PutBucketCorsAction, auth.PermissionWrite, nil
}
if queryArgs.Has("policy") {
// PutBucketPolicy
return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
}
if queryArgs.Has("acl") {
// PutBucketAcl
return auth.PutBucketAclAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousRequest)
}
// All the other rquestes are considered as 'CreateBucket' in the router
return "", "", s3err.GetAPIError(s3err.ErrAnonymousRequest)
}
if queryArgs.Has("tagging") {
// PutObjectTagging
return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
}
if queryArgs.Has("retention") {
// PutObjectRetention
return auth.PutObjectRetentionAction, auth.PermissionWrite, nil
}
if queryArgs.Has("legal-hold") {
// PutObjectLegalHold
return auth.PutObjectLegalHoldAction, auth.PermissionWrite, nil
}
if queryArgs.Has("acl") {
// PutObjectAcl
return auth.PutObjectAclAction, auth.PermissionWriteAcp, s3err.GetAPIError(s3err.ErrAnonymousRequest)
}
if queryArgs.Has("uploadId") && queryArgs.Has("partNumber") {
if ctx.Get("X-Amz-Copy-Source") != "" {
// UploadPartCopy
//TODO: Add public access check for copy-source
// Return AccessDenied for now
return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
}
// UploadPart
utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
return auth.PutObjectAction, auth.PermissionWrite, nil
}
if ctx.Get("X-Amz-Copy-Source") != "" {
return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousCopyObject)
}
// All the other requests are considered as 'PutObject' in the router
utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream())
return auth.PutObjectAction, auth.PermissionWrite, nil
case fiber.MethodPost:
if isBucketAction {
// DeleteObjects
// FIXME: should be fixed with https://github.com/versity/versitygw/issues/1327
// Return AccessDenied for now
return auth.DeleteObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied)
}
if queryArgs.Has("restore") {
return auth.RestoreObjectAction, auth.PermissionWrite, nil
}
if queryArgs.Has("select") && ctx.Query("select-type") == "2" {
// SelectObjectContent
return auth.GetObjectAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousRequest)
}
if queryArgs.Has("uploadId") {
// CompleteMultipartUpload
return auth.PutObjectAction, auth.PermissionWrite, nil
}
// All the other requests are considered as 'CreateMultipartUpload' in the router
return "", "", s3err.GetAPIError(s3err.ErrAnonymousCreateMp)
case fiber.MethodDelete:
if isBucketAction {
if queryArgs.Has("tagging") {
// DeleteBucketTagging
return auth.PutBucketTaggingAction, auth.PermissionWrite, nil
}
if queryArgs.Has("ownershipControls") {
// DeleteBucketOwnershipControls
return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership)
}
if queryArgs.Has("policy") {
// DeleteBucketPolicy
return auth.PutBucketPolicyAction, auth.PermissionWrite, nil
}
if queryArgs.Has("cors") {
// DeleteBucketCors
return auth.PutBucketCorsAction, auth.PermissionWrite, nil
}
// All the other requests are considered as 'DeleteBucket' in the router
return auth.DeleteBucketAction, auth.PermissionWrite, nil
}
if queryArgs.Has("tagging") {
// DeleteObjectTagging
return auth.PutObjectTaggingAction, auth.PermissionWrite, nil
}
if queryArgs.Has("uploadId") {
// AbortMultipartUpload
return auth.AbortMultipartUploadAction, auth.PermissionWrite, nil
}
// All the other requests are considered as 'DeleteObject' in the router
return auth.DeleteObjectAction, auth.PermissionWrite, nil
default:
// In no action is detected, return AccessDenied ?
return "", "", s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
// parsePath extracts the bucket and object names from the path
func parsePath(path string) (string, string) {
p := strings.TrimPrefix(path, "/")
bucket, object, _ := strings.Cut(p, "/")
return bucket, object
}

View File

@@ -0,0 +1,37 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
)
func SetDefaultValues(root RootUserConfig, region string) fiber.Handler {
return func(ctx *fiber.Ctx) error {
// These are necessary for the server access logs
utils.ContextKeyRegion.Set(ctx, region)
utils.ContextKeyStartTime.Set(ctx, time.Now())
utils.ContextKeyRootAccessKey.Set(ctx, root.Access)
// Set the account and isRoot to some defulat values, to avoid panics
// in case of public buckets
utils.ContextKeyAccount.Set(ctx, auth.Account{})
utils.ContextKeyIsRoot.Set(ctx, false)
return ctx.Next()
}
}

View File

@@ -83,11 +83,19 @@ func New(
app.Use(middlewares.HostStyleParser(server.virtualDomain))
}
// initilaze the default value setter middleware
app.Use(middlewares.SetDefaultValues(root, region))
// initialize the debug logger in debug mode
if server.debug {
app.Use(middlewares.DebugLogger())
}
app.Use(middlewares.ValidateBucketObjectNames(l, mm))
// Public buckets access checker
app.Use(middlewares.AuthorizePublicBucketAccess(be, l, mm))
// Authentication middlewares
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))

View File

@@ -88,11 +88,28 @@ func (c checksumType) isValid() bool {
c == checksumTypeCrc64nvme
}
// Extracts and validates the checksum type from the 'X-Amz-Trailer' header
func ExtractChecksumType(ctx *fiber.Ctx) (checksumType, error) {
trailer := ctx.Get("X-Amz-Trailer")
chType := checksumType(strings.ToLower(trailer))
if chType != "" && !chType.isValid() {
debuglogger.Logf("invalid value for 'X-Amz-Trailer': %v", chType)
return "", s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
}
return chType, nil
}
// IsSpecialPayload checks for special authorization types
func IsSpecialPayload(str string) bool {
return specialValues[payloadType(str)]
}
// Checks if the provided string is unsigned payload trailer type
func IsUnsignedStreamingPayload(str string) bool {
return payloadType(str) == payloadTypeStreamingUnsignedTrailer
}
// IsChunkEncoding checks for streaming/unsigned authorization types
func IsStreamingPayload(str string) bool {
pt := payloadType(str)
@@ -126,9 +143,12 @@ func NewChunkReader(ctx *fiber.Ctx, r io.Reader, authdata AuthData, region, secr
return nil, fmt.Errorf("invalid x-amz-content-sha256: %v", string(contentSha256))
}
checksumType := checksumType(strings.ToLower(ctx.Get("X-Amz-Trailer")))
if contentSha256 != payloadTypeStreamingSigned && !checksumType.isValid() {
debuglogger.Logf("invalid value for 'X-Amz-Trailer': %v", checksumType)
checksumType, err := ExtractChecksumType(ctx)
if err != nil {
return nil, err
}
if contentSha256 != payloadTypeStreamingSigned && checksumType == "" {
debuglogger.Logf("empty value for required trailer header 'X-Amz-Trailer': %v", checksumType)
return nil, s3err.GetAPIError(s3err.ErrTrailerHeaderNotSupported)
}

View File

@@ -0,0 +1,65 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"github.com/gofiber/fiber/v2"
)
// Region, StartTime, IsRoot, Account, AccessKey context locals
// are set to defualut values in middlewares.SetDefaultValues
// to avoid the nil interface conversions
type ContextKey string
const (
ContextKeyRegion ContextKey = "region"
ContextKeyStartTime ContextKey = "start-time"
ContextKeyIsRoot ContextKey = "is-root"
ContextKeyRootAccessKey ContextKey = "root-access-key"
ContextKeyAccount ContextKey = "account"
ContextKeyAuthenticated ContextKey = "authenticated"
ContextKeyPublicBucket ContextKey = "public-bucket"
ContextKeyParsedAcl ContextKey = "parsed-acl"
ContextKeySkipResBodyLog ContextKey = "skip-res-body-log"
ContextKeyBodyReader ContextKey = "body-reader"
)
func (ck ContextKey) Values() []ContextKey {
return []ContextKey{
ContextKeyRegion,
ContextKeyStartTime,
ContextKeyIsRoot,
ContextKeyRootAccessKey,
ContextKeyAccount,
ContextKeyAuthenticated,
ContextKeyPublicBucket,
ContextKeyParsedAcl,
ContextKeySkipResBodyLog,
ContextKeyBodyReader,
}
}
func (ck ContextKey) Set(ctx *fiber.Ctx, val any) {
ctx.Locals(string(ck), val)
}
func (ck ContextKey) IsSet(ctx *fiber.Ctx) bool {
val := ctx.Locals(string(ck))
return val != nil
}
func (ck ContextKey) Get(ctx *fiber.Ctx) any {
return ctx.Locals(string(ck))
}

View File

@@ -180,7 +180,7 @@ func ParsePresignedURIParts(ctx *fiber.Ctx) (AuthData, error) {
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
}
if ctx.Locals("region") != creds[2] {
if ContextKeyRegion.Get(ctx) != creds[2] {
return a, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),

View File

@@ -59,6 +59,11 @@ type ErrorCode int
const (
ErrNone ErrorCode = iota
ErrAccessDenied
ErrAnonymousRequest
ErrAnonymousCreateMp
ErrAnonymousCopyObject
ErrAnonymousPutBucketOwnership
ErrAnonymousGetBucketOwnership
ErrMethodNotAllowed
ErrBucketNotEmpty
ErrVersionedBucketNotEmpty
@@ -162,6 +167,7 @@ const (
ErrChecksumTypeWithAlgo
ErrInvalidChecksumHeader
ErrTrailerHeaderNotSupported
ErrBadRequest
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -186,6 +192,31 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Access Denied.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAnonymousRequest: {
Code: "AccessDenied",
Description: "Anonymous users cannot invoke this API. Please authenticate.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAnonymousCreateMp: {
Code: "AccessDenied",
Description: "Anonymous users cannot initiate multipart uploads. Please authenticate.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAnonymousCopyObject: {
Code: "AccessDenied",
Description: "Anonymous users cannot copy objects. Please authenticate.",
HTTPStatusCode: http.StatusForbidden,
},
ErrAnonymousPutBucketOwnership: {
Code: "AccessDenied",
Description: "s3:PutBucketOwnershipControls does not support Anonymous requests!",
HTTPStatusCode: http.StatusForbidden,
},
ErrAnonymousGetBucketOwnership: {
Code: "AccessDenied",
Description: "s3:GetBucketOwnershipControls does not support Anonymous requests!",
HTTPStatusCode: http.StatusForbidden,
},
ErrMethodNotAllowed: {
Code: "MethodNotAllowed",
Description: "The specified method is not allowed against this resource.",
@@ -696,6 +727,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The value specified in the x-amz-trailer header is not supported",
HTTPStatusCode: http.StatusBadRequest,
},
ErrBadRequest: {
Code: "400",
Description: "Bad Request",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {

View File

@@ -22,6 +22,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
)
type S3EventSender interface {
@@ -147,14 +148,14 @@ func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId)
bucket, object = path[1], strings.Join(path[2:], "/")
}
acc := ctx.Locals("account").(auth.Account)
acc := utils.ContextKeyAccount.Get(ctx).(auth.Account)
return EventSchema{
Records: []EventRecord{
{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
AwsRegion: utils.ContextKeyRegion.Get(ctx).(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{

View File

@@ -24,6 +24,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
)
@@ -74,7 +75,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
}
errorCode := ""
httpStatus := 200
startTime, ok := ctx.Locals("startTime").(time.Time)
startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
if !ok {
startTime = time.Now()
}
@@ -95,9 +96,9 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
}
}
switch ctx.Locals("account").(type) {
case auth.Account:
access = ctx.Locals("account").(auth.Account).Access
acct, ok := utils.ContextKeyAccount.Get(ctx).(auth.Account)
if ok {
access = acct.Access
}
lf.BucketOwner = meta.BucketOwner
@@ -121,7 +122,7 @@ func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", utils.ContextKeyRegion.Get(ctx).(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"

View File

@@ -22,6 +22,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
)
// FileLogger is a local file audit log
@@ -57,7 +58,10 @@ func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMe
access := "-"
reqURI := ctx.OriginalURL()
errorCode := ""
startTime := ctx.Locals("startTime").(time.Time)
startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
if !ok {
startTime = time.Now()
}
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
@@ -68,9 +72,9 @@ func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMe
errorCode = err.Error()
}
switch ctx.Locals("account").(type) {
switch utils.ContextKeyAccount.Get(ctx).(type) {
case auth.Account:
access = ctx.Locals("account").(auth.Account).Access
access = utils.ContextKeyAccount.Get(ctx).(auth.Account).Access
}
lf.Time = time.Now()

View File

@@ -28,6 +28,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
)
@@ -71,7 +72,10 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
}
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
startTime, ok := utils.ContextKeyStartTime.Get(ctx).(time.Time)
if !ok {
startTime = time.Now()
}
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
@@ -89,9 +93,9 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
}
}
switch ctx.Locals("account").(type) {
case auth.Account:
access = ctx.Locals("account").(auth.Account).Access
acct, ok := utils.ContextKeyAccount.Get(ctx).(auth.Account)
if ok {
access = acct.Access
}
lf.BucketOwner = meta.BucketOwner
@@ -115,7 +119,7 @@ func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMet
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", utils.ContextKeyRegion.Get(ctx).(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"

View File

@@ -15,7 +15,6 @@
package integration
func TestAuthentication(s *S3Conf) {
Authentication_empty_auth_header(s)
Authentication_invalid_auth_header(s)
Authentication_unsupported_signature_version(s)
Authentication_malformed_credentials(s)
@@ -37,7 +36,6 @@ func TestAuthentication(s *S3Conf) {
}
func TestPresignedAuthentication(s *S3Conf) {
PresignedAuth_missing_algo_query_param(s)
PresignedAuth_unsupported_algorithm(s)
PresignedAuth_missing_credentials_query_param(s)
PresignedAuth_malformed_creds_invalid_parts(s)
@@ -636,6 +634,11 @@ func TestFullFlow(s *S3Conf) {
TestGetObjectLegalHold(s)
TestWORMProtection(s)
TestAccessControl(s)
// FIXME: The tests should pass for azure as well
// but this issue should be fixed with https://github.com/versity/versitygw/issues/1336
if !s.azureTests {
TestPublicBuckets(s)
}
if s.versioningEnabled {
TestVersioning(s)
}
@@ -775,6 +778,13 @@ func TestAccessControl(s *S3Conf) {
AccessControl_copy_object_with_starting_slash_for_user(s)
}
func TestPublicBuckets(s *S3Conf) {
PublicBucket_default_privet_bucket(s)
PublicBucket_public_bucket_policy(s)
PublicBucket_public_object_policy(s)
PublicBucket_public_acl(s)
}
func TestVersioning(s *S3Conf) {
// PutBucketVersioning action
PutBucketVersioning_non_existing_bucket(s)
@@ -863,7 +873,6 @@ type IntTests map[string]func(s *S3Conf) error
func GetIntTests() IntTests {
return IntTests{
"Authentication_empty_auth_header": Authentication_empty_auth_header,
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
"Authentication_malformed_credentials": Authentication_malformed_credentials,
@@ -882,7 +891,6 @@ func GetIntTests() IntTests {
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
"Authentication_incorrect_md5": Authentication_incorrect_md5,
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
"PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param,
"PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm,
"PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param,
"PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts,
@@ -1277,6 +1285,10 @@ func GetIntTests() IntTests {
"AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl,
"AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access,
"AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user,
"PublicBucket_default_privet_bucket": PublicBucket_default_privet_bucket,
"PublicBucket_public_bucket_policy": PublicBucket_public_bucket_policy,
"PublicBucket_public_object_policy": PublicBucket_public_object_policy,
"PublicBucket_public_acl": PublicBucket_public_acl,
"PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket,
"PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status,
"PutBucketVersioning_success_enabled": PutBucketVersioning_success_enabled,

View File

@@ -130,16 +130,24 @@ func (c *S3Conf) GetClient() *s3.Client {
})
}
func (c *S3Conf) GetAnonymousClient() *s3.Client {
cfg := c.Config()
cfg.Credentials = aws.AnonymousCredentials{}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
if c.hostStyle {
o.BaseEndpoint = &c.endpoint
o.UsePathStyle = false
}
})
}
func (c *S3Conf) Config() aws.Config {
creds := c.getCreds()
tr := &http.Transport{}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(c.awsRegion),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
config.WithHTTPClient(c.httpClient),
config.WithRetryMaxAttempts(1),
}

File diff suppressed because it is too large Load Diff

View File

@@ -178,6 +178,8 @@ type setupCfg struct {
LockEnabled bool
VersioningStatus types.BucketVersioningStatus
Ownership types.ObjectOwnership
Anonymous bool
SkipTearDown bool
}
type setupOpt func(*setupCfg)
@@ -191,26 +193,47 @@ func withOwnership(o types.ObjectOwnership) setupOpt {
func withVersioning(v types.BucketVersioningStatus) setupOpt {
return func(s *setupCfg) { s.VersioningStatus = v }
}
func withAnonymousClient() setupOpt {
return func(s *setupCfg) { s.Anonymous = true }
}
func withSkipTearDown() setupOpt {
return func(s *setupCfg) { s.SkipTearDown = true }
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error, opts ...setupOpt) error {
runF(testName)
bucketName := getBucketName()
cfg := new(setupCfg)
for _, opt := range opts {
opt(cfg)
}
err := setup(s, bucketName, opts...)
if err != nil {
failF("%v: failed to create a bucket: %v", testName, err)
return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
}
client := s.GetClient()
var client *s3.Client
if cfg.Anonymous {
client = s.GetAnonymousClient()
} else {
client = s.GetClient()
}
handlerErr := handler(client, bucketName)
if handlerErr != nil {
failF("%v: %v", testName, handlerErr)
}
err = teardown(s, bucketName)
if err != nil {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
if handlerErr == nil {
return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
if !cfg.SkipTearDown {
err = teardown(s, bucketName)
if err != nil {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
if handlerErr == nil {
return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
}
}
}
if handlerErr == nil {
@@ -976,6 +999,52 @@ func genPolicyDoc(effect, principal, action, resource string) string {
return fmt.Sprintf(jsonTemplate, effect, principal, action, resource)
}
type policyType string
const (
policyTypeBucket policyType = "bucket"
policyTypeObject policyType = "object"
policyTypeFull policyType = "full"
)
func grantPublicBucketPolicy(client *s3.Client, bucket string, tp policyType) error {
var doc string
switch tp {
case policyTypeBucket:
doc = genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%s"`, bucket))
case policyTypeObject:
doc = genPolicyDoc("Allow", `"*"`, `"s3:*"`, fmt.Sprintf(`"arn:aws:s3:::%s/*"`, bucket))
case policyTypeFull:
template := `
{
"Statement": [
{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": "arn:aws:s3:::%s"
},
{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": "arn:aws:s3:::%s/*"
}
]
}
`
doc = fmt.Sprintf(template, bucket, bucket)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: &bucket,
Policy: &doc,
})
cancel()
return err
}
func getMalformedPolicyError(msg string) s3err.APIError {
return s3err.APIError{
Code: "MalformedPolicy",
@@ -1334,3 +1403,9 @@ func checkObjectMetaProps(client *s3.Client, bucket, object string, o ObjectMeta
func getBoolPtr(b bool) *bool {
return &b
}
type PublicBucketTestCase struct {
Action string
Call func(ctx context.Context) error
ExpectedErr error
}