From 394675a5a8243f04e2b1e514d5a9066557ff1af2 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 16 Jul 2025 00:14:29 +0400 Subject: [PATCH] feat: implements unit tests for controller utilities --- metrics/metrics.go | 24 +- s3api/controllers/admin_test.go | 27 + s3api/controllers/base.go | 1890 +------------------------- s3api/controllers/base_test.go | 576 +++++++- s3api/controllers/bucket-put.go | 36 +- s3api/controllers/bucket-put_test.go | 290 +++- s3api/controllers/object-get.go | 4 +- s3api/router.go | 2 +- s3api/server.go | 2 +- s3api/utils/utils.go | 9 + 10 files changed, 949 insertions(+), 1911 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 22cad81..753b405 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -41,8 +41,14 @@ type Tag struct { Value string } -// Manager is a manager of metrics plugins -type Manager struct { +// Manager is the interface definition for metrics manager +type Manager interface { + Send(ctx *fiber.Ctx, err error, action string, count int64, status int) + Close() +} + +// manager is a manager of metrics plugins +type manager struct { wg sync.WaitGroup ctx context.Context @@ -59,7 +65,7 @@ type Config struct { } // NewManager initializes metrics plugins and returns a new metrics manager -func NewManager(ctx context.Context, conf Config) (*Manager, error) { +func NewManager(ctx context.Context, conf Config) (Manager, error) { if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 { return nil, nil } @@ -74,7 +80,7 @@ func NewManager(ctx context.Context, conf Config) (*Manager, error) { addDataChan := make(chan datapoint, dataItemCount) - mgr := &Manager{ + mgr := &manager{ addDataChan: addDataChan, ctx: ctx, config: conf, @@ -112,7 +118,7 @@ func NewManager(ctx context.Context, conf Config) (*Manager, error) { return mgr, nil } -func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) { +func (m *manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) { // In case of Authentication failures, url parsing ... if action == "" { action = ActionUndetected @@ -168,12 +174,12 @@ func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, st } // increment increments the key by one -func (m *Manager) increment(key string, tags ...Tag) { +func (m *manager) increment(key string, tags ...Tag) { m.add(key, 1, tags...) } // add adds value to key -func (m *Manager) add(key string, value int64, tags ...Tag) { +func (m *manager) add(key string, value int64, tags ...Tag) { if m.ctx.Err() != nil { return } @@ -192,7 +198,7 @@ func (m *Manager) add(key string, value int64, tags ...Tag) { } // Close closes metrics channels, waits for data to complete, closes all plugins -func (m *Manager) Close() { +func (m *manager) Close() { // drain the datapoint channels close(m.addDataChan) m.wg.Wait() @@ -209,7 +215,7 @@ type publisher interface { Close() } -func (m *Manager) addForwarder(addChan <-chan datapoint) { +func (m *manager) addForwarder(addChan <-chan datapoint) { for data := range addChan { for _, s := range m.publishers { s.Add(data.key, data.value, data.tags...) diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go index 4469620..e38b79f 100644 --- a/s3api/controllers/admin_test.go +++ b/s3api/controllers/admin_test.go @@ -23,11 +23,38 @@ import ( "github.com/stretchr/testify/assert" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" "github.com/versity/versitygw/s3response" ) +func TestNewAdminController(t *testing.T) { + type args struct { + iam auth.IAMService + be backend.Backend + l s3log.AuditLogger + } + tests := []struct { + name string + args args + want AdminController + }{ + { + name: "initialize admin api", + args: args{}, + want: AdminController{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewAdminController(tt.args.iam, tt.args.be, tt.args.l) + assert.Equal(t, got, tt.want) + }) + } +} + func TestAdminController_CreateUser(t *testing.T) { validBody, err := xml.Marshal(auth.Account{ Access: "access", diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 1ccf4e5..f788026 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -16,18 +16,10 @@ package controllers import ( "encoding/xml" - "errors" "fmt" - "io" "net/http" - "net/url" "os" - "strconv" - "strings" - "time" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" @@ -37,7 +29,6 @@ import ( "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" - "github.com/versity/versitygw/s3response" ) type S3ApiController struct { @@ -45,7 +36,7 @@ type S3ApiController struct { iam auth.IAMService logger s3log.AuditLogger evSender s3event.S3EventSender - mm *metrics.Manager + mm metrics.Manager debug bool readonly bool } @@ -61,7 +52,7 @@ var ( xmlhdr = []byte(`` + "\n") ) -func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController { +func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm metrics.Manager, debug bool, readonly bool) S3ApiController { if debug { debuglogger.SetDebugEnabled() } @@ -82,1869 +73,8 @@ func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { return &Response{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) } -func getint64(i *int64) int64 { - if i == nil { - return 0 - } - return *i -} - -func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acl := ctx.Get("X-Amz-Acl") - grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") - grantRead := ctx.Get("X-Amz-Grant-Read") - grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") - grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - objectOwnership := types.ObjectOwnership( - ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), - ) - // mfa := ctx.Get("X-Amz-Mfa") - // contentMD5 := ctx.Get("Content-MD5") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - - if ctx.Request().URI().QueryArgs().Has("tagging") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - - tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitBucket) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketTaggingAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketTagging(ctx.Context(), bucket, tagging) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - if ctx.Request().URI().QueryArgs().Has("ownershipControls") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var ownershipControls s3response.OwnershipControls - if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil { - if c.debug { - debuglogger.Logf("failed to unmarshal request body: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - rulesCount := len(ownershipControls.Rules) - isValidOwnership := utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) - if rulesCount != 1 || !isValidOwnership { - if c.debug && rulesCount != 1 { - debuglogger.Logf("ownership control rules should be 1, got %v", rulesCount) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketOwnershipControlsAction, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("versioning") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketVersioningAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - var versioningConf types.VersioningConfiguration - err = xml.Unmarshal(ctx.Body(), &versioningConf) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling versioning configuration: %v", - err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - if versioningConf.Status != types.BucketVersioningStatusEnabled && - versioningConf.Status != types.BucketVersioningStatusSuspended { - if c.debug { - debuglogger.Logf("invalid versioning configuration status: %v", versioningConf.Status) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("object-lock") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketObjectLockConfigurationAction, - IsBucketPublic: isPublicBucket, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - config, err := auth.ParseBucketLockConfigurationInput(ctx.Body()) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectLockConfiguration(ctx.Context(), bucket, config) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("cors") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketCorsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketCors(ctx.Context(), []byte{}) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketCors, - BucketOwner: parsedAcl.Owner, - }, - ) - } - } - if ctx.Request().URI().QueryArgs().Has("policy") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketPolicyAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }, - ) - } - - err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body()) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP - - if ctx.Request().URI().QueryArgs().Has("acl") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var input *auth.PutBucketAclInput - - ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) - if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if ownership == types.ObjectOwnershipBucketOwnerEnforced { - if c.debug { - debuglogger.Logf("bucket acls are disabled") - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAclNotSupported), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWriteAcp, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketAclAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if len(ctx.Body()) > 0 { - var accessControlPolicy auth.AccessControlPolicy - err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling access control policy: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedACL), - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = accessControlPolicy.Validate() - if err != nil { - if c.debug { - debuglogger.Logf("invalid access control policy: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if *accessControlPolicy.Owner.ID != parsedAcl.Owner { - if c.debug { - debuglogger.Logf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner) - } - return SendResponse(ctx, s3err.APIError{ - Code: "InvalidArgument", - Description: "Invalid id", - HTTPStatusCode: http.StatusBadRequest, - }, - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if grants+acl != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrUnexpectedContent), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &accessControlPolicy, - } - } else if acl != "" { - if acl != "private" && acl != "public-read" && acl != "public-read-write" { - if c.debug { - debuglogger.Logf("invalid acl: %q", acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - ACL: types.BucketCannedACL(acl), - } - } else if grants != "" { - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - } - } else { - if c.debug { - debuglogger.Logf("none of the bucket acl options has been specified: canned, req headers, req body") - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrMissingSecurityHeader), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam, acct.Role == auth.RoleAdmin) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if ok := utils.IsValidOwnership(objectOwnership); !ok { - return SendResponse(ctx, s3err.APIError{ - Code: "InvalidArgument", - Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership), - HTTPStatusCode: http.StatusBadRequest, - }, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced { - if c.debug { - debuglogger.Logf("bucket acls are disabled for %v object ownership", objectOwnership) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - if acl != "" && grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - defACL := auth.ACL{ - Owner: acct.Access, - } - - updAcl, err := auth.UpdateACL(&auth.PutBucketAclInput{ - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - AccessControlPolicy: &auth.AccessControlPolicy{ - Owner: &types.Owner{ - ID: &acct.Access, - }}, - ACL: types.BucketCannedACL(acl), - }, defACL, c.iam, acct.Role == auth.RoleAdmin) - if err != nil { - if c.debug { - debuglogger.Logf("failed to update bucket acl: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - lockHeader := ctx.Get("X-Amz-Bucket-Object-Lock-Enabled") - // CLI provides "True", SDK - "true" - lockEnabled := lockHeader == "True" || lockHeader == "true" - - err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{ - Bucket: &bucket, - ObjectOwnership: objectOwnership, - ObjectLockEnabledForBucket: &lockEnabled, - }, updAcl) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) -} - -func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - keyStart := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - versionId := ctx.Query("versionId") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - contentType := ctx.Get("Content-Type") - contentEncoding := ctx.Get("Content-Encoding") - contentDisposition := ctx.Get("Content-Disposition") - contentLanguage := ctx.Get("Content-Language") - cacheControl := ctx.Get("Cache-Control") - expires := ctx.Get("Expires") - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - tagging := ctx.Get("x-amz-tagging") - - // Copy source headers - copySource := ctx.Get("X-Amz-Copy-Source") - if len(copySource) > 0 && copySource[0] == '/' { - copySource = copySource[1:] - } - copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") - copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") - copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") - copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") - copySrcRange := ctx.Get("X-Amz-Copy-Source-Range") - directive := ctx.Get("X-Amz-Metadata-Directive") - - // Permission headers - acl := ctx.Get("X-Amz-Acl") - grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") - grantRead := ctx.Get("X-Amz-Grant-Read") - grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") - grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - - // Content Length - contentLengthStr := ctx.Get("Content-Length") - if contentLengthStr == "" { - contentLengthStr = "0" - } - // Use decoded content length if available because the - // middleware will decode the chunked transfer encoding - decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") - if decodedLength != "" { - contentLengthStr = decodedLength - } - - // Other headers - bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner") - storageClass := ctx.Get("X-Amz-Storage-Class") - - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP - - if keyEnd != "" { - keyStart = strings.Join([]string{keyStart, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && keyStart[len(keyStart)-1:] != "/" { - keyStart = keyStart + "/" - } - - if ctx.Request().URI().QueryArgs().Has("tagging") { - tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutBucketTaggingAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectTagging(ctx.Context(), bucket, keyStart, tagging) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectTaggingPut, - }) - } - - if ctx.Request().URI().QueryArgs().Has("retention") { - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectRetentionAction, - IsBucketPublic: IsBucketPublic, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - bypass := strings.EqualFold(bypassHdr, "true") - if bypass { - policy, err := c.be.GetBucketPolicy(ctx.Context(), bucket) - if err != nil { - bypass = false - } else { - if err := auth.VerifyBucketPolicy(policy, acct.Access, bucket, keyStart, auth.BypassGovernanceRetentionAction); err != nil { - bypass = false - } - } - } - - retention, err := auth.ParseObjectLockRetentionInput(ctx.Body()) - if err != nil { - if c.debug { - debuglogger.Logf("failed to parse object lock configuration input: %v", err) - } - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectRetention(ctx.Context(), bucket, keyStart, versionId, bypass, retention) - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("legal-hold") { - var legalHold types.ObjectLockLegalHold - if err := xml.Unmarshal(ctx.Body(), &legalHold); err != nil { - if c.debug { - debuglogger.Logf("failed to parse request body: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if legalHold.Status != types.ObjectLockLegalHoldStatusOff && legalHold.Status != types.ObjectLockLegalHoldStatusOn { - if c.debug { - debuglogger.Logf("invalid legal hold status: %v", legalHold.Status) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectLegalHoldAction, - IsBucketPublic: IsBucketPublic, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - err := c.be.PutObjectLegalHold(ctx.Context(), bucket, keyStart, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("uploadId") && - ctx.Request().URI().QueryArgs().Has("partNumber") && - copySource != "" { - - cs := copySource - copySource, err := url.QueryUnescape(copySource) - if err != nil { - if c.debug { - debuglogger.Logf("error unescaping copy source %q: %v", - cs, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - partNumber := int32(ctx.QueryInt("partNumber", -1)) - if partNumber < 1 || partNumber > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumber) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, - auth.AccessOptions{ - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - resp, err := c.be.UploadPartCopy(ctx.Context(), - &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &keyStart, - CopySource: ©Source, - PartNumber: &partNumber, - UploadId: &uploadId, - ExpectedBucketOwner: &bucketOwner, - CopySourceRange: ©SrcRange, - }) - if err == nil && resp.CopySourceVersionId != "" { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-copy-source-version-id", - Value: resp.CopySourceVersionId, - }, - }) - } - return SendXMLResponse(ctx, resp, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("uploadId") && - ctx.Request().URI().QueryArgs().Has("partNumber") { - partNumber := int32(ctx.QueryInt("partNumber", -1)) - if partNumber < 1 || partNumber > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumber) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing content length %q: %v", - contentLengthStr, err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) - if err != nil { - if c.debug { - debuglogger.Logf("err parsing checksum headers: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var body io.Reader - bodyi := utils.ContextKeyBodyReader.Get(ctx) - if bodyi != nil { - body = bodyi.(io.Reader) - } else { - body = ctx.Request().BodyStream() - } - - res, err := c.be.UploadPart(ctx.Context(), - &s3.UploadPartInput{ - Bucket: &bucket, - Key: &keyStart, - UploadId: &uploadId, - PartNumber: &partNumber, - ContentLength: &contentLength, - Body: body, - ChecksumAlgorithm: algorithm, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), - }) - if err == nil { - headers := []utils.CustomHeader{} - if res.ETag != nil { - headers = append(headers, utils.CustomHeader{ - Key: "ETag", - Value: *res.ETag, - }) - } - switch { - case res.ChecksumCRC32 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - - utils.SetResponseHeaders(ctx, headers) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("acl") { - var input *s3.PutObjectAclInput - - if len(ctx.Body()) > 0 { - if grants+acl != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - var accessControlPolicy auth.AccessControlPolicy - err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling access control policy: %v", - err) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - //TODO: This part will be changed when object acls are implemented - - grants := []types.Grant{} - for _, grt := range accessControlPolicy.AccessControlList.Grants { - grants = append(grants, types.Grant{ - Grantee: &types.Grantee{ - ID: &grt.Grantee.ID, - Type: grt.Grantee.Type, - }, - Permission: types.Permission(grt.Permission), - }) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - ACL: "", - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: accessControlPolicy.Owner, - Grants: grants, - }, - } - } - if acl != "" { - if acl != "private" && acl != "public-read" && acl != "public-read-write" { - if c.debug { - debuglogger.Logf("invalid acl: %q", acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if len(ctx.Body()) > 0 || grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", - grants, acl, len(ctx.Body())) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - ACL: types.ObjectCannedACL(acl), - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &bucketOwner}, - }, - } - } - if grants != "" { - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &bucketOwner}, - }, - ACL: "", - } - } - - err := c.be.PutObjectAcl(ctx.Context(), input) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectAclPut, - }) - } - - if copySource != "" { - cs := copySource - copySource, err := url.QueryUnescape(copySource) - if err != nil { - if c.debug { - debuglogger.Logf("error unescaping copy source %q: %v", - cs, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, - auth.AccessOptions{ - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var mtime *time.Time - var umtime *time.Time - if copySrcModifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcModifSince) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing copy source modified since %q: %v", - copySrcModifSince, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - mtime = &tm - } - if copySrcUnmodifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing copy source unmodified since %q: %v", - copySrcUnmodifSince, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - umtime = &tm - } - - metadata := utils.GetUserMetaData(&ctx.Request().Header) - - if directive != "" && directive != "COPY" && directive != "REPLACE" { - if c.debug { - debuglogger.Logf("invalid metadata directive: %v", directive) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidMetadataDirective), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - metaDirective := types.MetadataDirectiveCopy - if directive == "REPLACE" { - metaDirective = types.MetadataDirectiveReplace - } - - tDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive")) - if tDirective != "" && tDirective != types.TaggingDirectiveCopy && tDirective != types.TaggingDirectiveReplace { - if c.debug { - debuglogger.Logf("invalid tagging direcrive: %v", tDirective) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidTaggingDirective), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - taggingDirective := types.TaggingDirectiveCopy - if tDirective == types.TaggingDirectiveReplace { - taggingDirective = types.TaggingDirectiveReplace - } - - checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) - err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - objLock, err := utils.ParsObjectLockHdrs(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.CopyObject(ctx.Context(), - s3response.CopyObjectInput{ - Bucket: &bucket, - Key: &keyStart, - ContentType: &contentType, - ContentDisposition: &contentDisposition, - ContentEncoding: &contentEncoding, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - Tagging: &tagging, - TaggingDirective: taggingDirective, - CopySource: ©Source, - CopySourceIfMatch: ©SrcIfMatch, - CopySourceIfNoneMatch: ©SrcIfNoneMatch, - CopySourceIfModifiedSince: mtime, - CopySourceIfUnmodifiedSince: umtime, - ExpectedBucketOwner: &acct.Access, - Metadata: metadata, - MetadataDirective: metaDirective, - StorageClass: types.StorageClass(storageClass), - ChecksumAlgorithm: checksumAlgorithm, - ObjectLockRetainUntilDate: &objLock.RetainUntilDate, - ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, - ObjectLockMode: objLock.ObjectLockMode, - }) - if err == nil { - hdrs := []utils.CustomHeader{} - if getstring(res.VersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - if getstring(res.CopySourceVersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-copy-source-version-id", - Value: getstring(res.CopySourceVersionId), - }) - } - utils.SetResponseHeaders(ctx, hdrs) - - return SendXMLResponse(ctx, res.CopyObjectResult, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - ObjectETag: res.CopyObjectResult.ETag, - VersionId: res.VersionId, - EventName: s3event.EventObjectCreatedCopy, - }) - } else { - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - } - - metadata := utils.GetUserMetaData(&ctx.Request().Header) - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing content length %q: %v", - contentLengthStr, err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - objLock, err := utils.ParsObjectLockHdrs(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var body io.Reader - bodyi := utils.ContextKeyBodyReader.Get(ctx) - if bodyi != nil { - body = bodyi.(io.Reader) - } else { - body = ctx.Request().BodyStream() - } - - res, err := c.be.PutObject(ctx.Context(), - s3response.PutObjectInput{ - Bucket: &bucket, - Key: &keyStart, - ContentLength: &contentLength, - ContentType: &contentType, - ContentEncoding: &contentEncoding, - ContentDisposition: &contentDisposition, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - Metadata: metadata, - Body: body, - Tagging: &tagging, - ObjectLockRetainUntilDate: &objLock.RetainUntilDate, - ObjectLockMode: objLock.ObjectLockMode, - ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, - ChecksumAlgorithm: algorithm, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - EvSender: c.evSender, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - ObjectSize: contentLength, - EventName: s3event.EventObjectCreatedPut, - }) - } - hdrs := []utils.CustomHeader{ - { - Key: "ETag", - Value: res.ETag, - }, - } - - if res.VersionID != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: res.VersionID, - }) - } - switch { - case res.ChecksumCRC32 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - if res.ChecksumType != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-type", - Value: string(res.ChecksumType), - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - EvSender: c.evSender, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - ObjectETag: &res.ETag, - ObjectSize: contentLength, - EventName: s3event.EventObjectCreatedPut, - }) -} - -func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - var dObj s3response.DeleteObjects - - err := xml.Unmarshal(ctx.Body(), &dObj) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling delete objects: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.DeleteObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - // The AWS CLI sends 'True', while Go SDK sends 'true' - bypass := strings.EqualFold(bypassHdr, "true") - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.DeleteObjects(ctx.Context(), - &s3.DeleteObjectsInput{ - Bucket: &bucket, - Delete: &types.Delete{ - Objects: dObj.Objects, - }, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - ObjectCount: int64(len(dObj.Objects)), - BucketOwner: parsedAcl.Owner, - EvSender: c.evSender, - EventName: s3event.EventObjectRemovedDeleteObjects, - }) -} - -func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - versionId := ctx.Query("versionId") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - if ctx.Request().URI().QueryArgs().Has("tagging") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.DeleteObjectTaggingAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteObjectTagging(ctx.Context(), bucket, key) - return SendResponse(ctx, err, - &MetaOpts{ - Status: http.StatusNoContent, - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObjectTagging, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectTaggingDelete, - }) - } - - if uploadId != "" { - expectedBucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner") - requestPayer := ctx.Get("X-Amz-Request-Payer") - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.AbortMultipartUploadAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionAbortMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.AbortMultipartUpload(ctx.Context(), - &s3.AbortMultipartUploadInput{ - UploadId: &uploadId, - Bucket: &bucket, - Key: &key, - ExpectedBucketOwner: &expectedBucketOwner, - RequestPayer: types.RequestPayer(requestPayer), - }) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionAbortMultipartUpload, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.DeleteObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - }) - } - - // The AWS CLI sends 'True', while Go SDK sends 'true' - bypass := strings.EqualFold(bypassHdr, "true") - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.DeleteObject(ctx.Context(), - &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &key, - VersionId: &versionId, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectRemovedDelete, - Status: http.StatusNoContent, - }) - } - - hdrs := []utils.CustomHeader{} - if res.VersionId != nil && *res.VersionId != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: *res.VersionId, - }) - } - if res.DeleteMarker != nil && *res.DeleteMarker { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-delete-marker", - Value: "true", - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectRemovedDelete, - Status: http.StatusNoContent, - }) -} - -func (c S3ApiController) HeadBuckets(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - region := utils.ContextKeyRegion.Get(ctx).(string) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadBucket, - BucketOwner: parsedAcl.Owner, - }) - } - - _, err = c.be.HeadBucket(ctx.Context(), - &s3.HeadBucketInput{ - Bucket: &bucket, - }) - - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "X-Amz-Access-Point-Alias", - Value: "false", - }, - { - Key: "X-Amz-Bucket-Region", - Value: region, - }, - }) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadBucket, - BucketOwner: parsedAcl.Owner, - }) -} - -const ( - timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" -) - -type MetaOpts struct { - Logger s3log.AuditLogger - EvSender s3event.S3EventSender - MetricsMng *metrics.Manager +// MetaOptions holds the metadata for metrics, audit logs and s3 events +type MetaOptions struct { ContentLength int64 BucketOwner string ObjectSize int64 @@ -1969,7 +99,7 @@ type Response struct { type Services struct { Logger s3log.AuditLogger EventSender s3event.S3EventSender - MetricsManager *metrics.Manager + MetricsManager metrics.Manager } // Controller is the type definition for an s3api controller @@ -2001,7 +131,7 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand // WrapMiddleware executes the given middleware and handles sending the audit logs // and metrics. It also handles the error parsing -func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { +func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.Manager) fiber.Handler { return func(ctx *fiber.Ctx) error { err := handler(ctx) if err != nil { @@ -2094,11 +224,12 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s responseBytes = encodedResp } else { if responseBytes, err = xml.Marshal(response.Data); err != nil { - return err + debuglogger.Logf("Internal Error, %v", err) + return ctx.Status(http.StatusInternalServerError).Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) } if len(responseBytes) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) } } @@ -2142,6 +273,9 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s res = append(res, xmlhdr...) res = append(res, responseBytes...) + // Set the Content-Length header + ctx.Response().Header.SetContentLength(msglen) + return ctx.Send(res) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 5cabf37..ed62aaf 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -16,17 +16,27 @@ package controllers import ( "bytes" + "encoding/xml" + "errors" + "fmt" "net/http" "net/http/httptest" "net/url" "path" + "strings" "testing" "github.com/gofiber/fiber/v2" "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3log" + "github.com/versity/versitygw/s3response" ) var ( @@ -143,8 +153,572 @@ func buildRequest(bucket, object string, body []byte, headers, queries map[strin // set the request headers for key, val := range headers { - req.Header.Add(key, val) + req.Header.Set(key, val) } return req } + +func TestNew(t *testing.T) { + type args struct { + be backend.Backend + iam auth.IAMService + logger s3log.AuditLogger + evs s3event.S3EventSender + mm metrics.Manager + debug bool + readonly bool + } + tests := []struct { + name string + args args + want S3ApiController + }{ + { + name: "debug enabled", + args: args{ + debug: true, + }, + want: S3ApiController{ + debug: true, + }, + }, + { + name: "debug disabled", + args: args{ + debug: false, + }, + want: S3ApiController{ + debug: false, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := New(tt.args.be, tt.args.iam, tt.args.logger, tt.args.evs, tt.args.mm, tt.args.debug, tt.args.readonly) + assert.Equal(t, got, tt.want) + }) + } +} + +func TestS3ApiController_HandleUnmatch(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "return method not allowed", + output: testOutput{ + response: &Response{}, + err: s3err.GetAPIError(s3err.ErrMethodNotAllowed), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := S3ApiController{} + + testController( + t, + ctrl.HandleUnmatch, + tt.output.response, + tt.output.err, + ctxInputs{}) + }) + } +} + +func TestSetResponseHeaders(t *testing.T) { + type args struct { + headers map[string]*string + } + tests := []struct { + name string + args args + expected map[string]string + }{ + { + name: "should not set if map is nil", + args: args{ + headers: nil, + }, + expected: nil, + }, + { + name: "should set some headers", + args: args{ + headers: map[string]*string{ + "x-amz-checksum-algorithm": utils.GetStringPtr("crc32"), + "x-amz-meta-key": utils.GetStringPtr("meta_key"), + "x-amz-mp-size": utils.GetStringPtr(""), + "something": nil, + }, + }, + expected: map[string]string{ + "x-amz-checksum-algorithm": "crc32", + "x-amz-meta-key": "meta_key", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := fiber.New() + ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) + SetResponseHeaders(ctx, tt.args.headers) + if tt.expected != nil { + for key, val := range tt.expected { + v := ctx.Response().Header.Peek(key) + assert.Equal(t, val, string(v)) + } + } + }) + } +} + +// mock the audit logger +type mockAuditLogger struct { +} + +func (m *mockAuditLogger) Log(_ *fiber.Ctx, _ error, _ []byte, _ s3log.LogMeta) {} +func (m *mockAuditLogger) HangUp() error { return nil } +func (m *mockAuditLogger) Shutdown() error { return nil } + +// mock S3 event sender +type mockEvSender struct { +} + +func (m *mockEvSender) SendEvent(_ *fiber.Ctx, _ s3event.EventMeta) {} +func (m *mockEvSender) Close() error { return nil } + +// mock metrics manager + +type mockMetricsManager struct{} + +func (m *mockMetricsManager) Send(_ *fiber.Ctx, _ error, _ string, _ int64, _ int) {} +func (m *mockMetricsManager) Close() {} + +func TestProcessController(t *testing.T) { + payload, err := xml.Marshal(s3response.Bucket{ + Name: "something", + }) + assert.NoError(t, err) + + payloadLen := len(payload) + len(xmlhdr) + + services := &Services{ + Logger: &mockAuditLogger{}, + EventSender: &mockEvSender{}, + MetricsManager: &mockMetricsManager{}, + } + type args struct { + controller Controller + svc *Services + } + type expected struct { + status int + headers map[string]string + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "no services successfull response", + args: args{ + svc: &Services{}, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, nil + }, + }, + expected: expected{ + status: http.StatusOK, + }, + }, + { + name: "handle api error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + }, + }, + expected: expected{ + status: http.StatusBadRequest, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInvalidRequest), "", "", ""), + }, + }, + { + name: "handle custom error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, errors.New("custom error") + }, + }, + expected: expected{ + status: http.StatusInternalServerError, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + { + name: "body parsing fails", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: make(chan int), + }, nil + }, + }, + expected: expected{ + status: http.StatusInternalServerError, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + { + name: "no data payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{ + ObjectCount: 2, + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + }, + }, + { + name: "should return 204 http status", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{ + Status: http.StatusNoContent, + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusNoContent, + }, + }, + { + name: "already encoded payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: []byte("encoded_data"), + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + body: []byte("encoded_data"), + headers: map[string]string{ + "Content-Length": "12", + }, + }, + }, + { + name: "should set response headers", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Headers: map[string]*string{ + "X-Amz-My-Custom-Header": utils.GetStringPtr("my_value"), + "X-Amz-Meta-My-Meta": utils.GetStringPtr("my_meta"), + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + headers: map[string]string{ + "X-Amz-My-Custom-Header": "my_value", + "X-Amz-Meta-My-Meta": "my_meta", + }, + }, + }, + { + name: "large paylod: should return internal error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + type Item struct { + Value string `xml:"value"` + } + + type payload struct { + Items []Item `xml:"item"` + } + + const targetSize = 5 * 1024 * 1024 // 5 MiB + const itemCount = 500 + const valueSize = targetSize / itemCount + + p := payload{ + Items: make([]Item, itemCount), + } + + // Preallocate one shared string of desired size + var sb strings.Builder + sb.Grow(valueSize) + for range valueSize { + sb.WriteByte('A') + } + largeValue := sb.String() + + for i := range p.Items { + p.Items[i].Value = largeValue + } + + return &Response{ + Data: p, + }, nil + }, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + status: http.StatusInternalServerError, + }, + }, + { + name: "not encoded payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: s3response.Bucket{ + Name: "something", + }, + }, nil + }, + }, + expected: expected{ + headers: map[string]string{ + "Content-Length": fmt.Sprint(payloadLen), + }, + body: append(xmlhdr, payload...), + status: http.StatusOK, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := fiber.New().AcquireCtx(&fasthttp.RequestCtx{}) + err := ProcessController(ctx, tt.args.controller, metrics.ActionAbortMultipartUpload, tt.args.svc) + assert.NoError(t, err) + + // check the status + assert.Equal(t, tt.expected.status, ctx.Response().StatusCode()) + + // check the response headers to be set + if tt.expected.headers != nil { + for key, val := range tt.expected.headers { + v := ctx.Response().Header.Peek(key) + assert.Equal(t, val, string(v)) + } + } + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + }) + } +} + +func TestProcessHandlers(t *testing.T) { + payload, err := xml.Marshal(s3response.Checksum{ + CRC32: utils.GetStringPtr("crc32"), + }) + assert.NoError(t, err) + + type args struct { + controller Controller + svc *Services + handlers []fiber.Handler + locals map[utils.ContextKey]any + } + type expected struct { + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "should skip the handlers", + args: args{ + locals: map[utils.ContextKey]any{ + utils.ContextKeySkip: true, + }, + }, + }, + { + name: "handler returns error", + args: args{ + handlers: []fiber.Handler{ + func(ctx *fiber.Ctx) error { + return nil + }, + func(ctx *fiber.Ctx) error { + return s3err.GetAPIError(s3err.ErrAccessDenied) + }, + }, + svc: &Services{}, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAccessDenied), "", "", ""), + }, + }, + { + name: "should process the controller", + args: args{ + handlers: []fiber.Handler{ + func(ctx *fiber.Ctx) error { + return nil + }, + func(ctx *fiber.Ctx) error { + return nil + }, + }, + svc: &Services{}, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: s3response.Checksum{ + CRC32: utils.GetStringPtr("crc32"), + }, + }, nil + }, + }, + expected: expected{ + body: append(xmlhdr, payload...), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mdlwr := ProcessHandlers(tt.args.controller, metrics.ActionCreateBucket, tt.args.svc, tt.args.handlers...) + + app := fiber.New() + + app.Post("/:bucket/*", func(ctx *fiber.Ctx) error { + // set the request locals + if tt.args.locals != nil { + for key, val := range tt.args.locals { + key.Set(ctx, val) + } + } + + // call the controller by passing the ctx + err := mdlwr(ctx) + assert.NoError(t, err) + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + + return nil + }) + + app.All("*", func(ctx *fiber.Ctx) error { + return nil + }) + + req := buildRequest("bucket", "object", nil, nil, nil) + + _, err := app.Test(req) + assert.NoError(t, err) + }) + } +} + +func TestWrapMiddleware(t *testing.T) { + type args struct { + handler fiber.Handler + logger s3log.AuditLogger + mm metrics.Manager + } + type expected struct { + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "handler returns no error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return nil + }, + }, + }, + { + name: "handler returns api error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return s3err.GetAPIError(s3err.ErrAclNotSupported) + }, + mm: &mockMetricsManager{}, + logger: &mockAuditLogger{}, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAclNotSupported), "", "", ""), + }, + }, + { + name: "handler returns custom error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return errors.New("custom error") + }, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mdlwr := WrapMiddleware(tt.args.handler, tt.args.logger, tt.args.mm) + app := fiber.New() + + app.Post("/:bucket/*", func(ctx *fiber.Ctx) error { + // call the controller by passing the ctx + err := mdlwr(ctx) + assert.NoError(t, err) + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + + return nil + }) + + app.All("*", func(ctx *fiber.Ctx) error { + return nil + }) + + req := buildRequest("bucket", "object", nil, nil, nil) + + _, err := app.Test(req) + assert.NoError(t, err) + }) + } +} diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index 6080c06..8a0caf4 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -311,24 +311,7 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP var input *auth.PutBucketAclInput - ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) - if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - if ownership == types.ObjectOwnershipBucketOwnerEnforced { - debuglogger.Logf("bucket acls are disabled") - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrAclNotSupported) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -346,6 +329,23 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { }, err } + ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + if ownership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAclNotSupported) + } + if len(ctx.Body()) > 0 { var accessControlPolicy auth.AccessControlPolicy err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) diff --git a/s3api/controllers/bucket-put_test.go b/s3api/controllers/bucket-put_test.go index 622354a..239addc 100644 --- a/s3api/controllers/bucket-put_test.go +++ b/s3api/controllers/bucket-put_test.go @@ -17,6 +17,7 @@ package controllers import ( "context" "encoding/xml" + "errors" "fmt" "net/http" "testing" @@ -810,4 +811,291 @@ func TestS3ApiController_CreateBucket(t *testing.T) { } } -// TODO: add a test for PutBucketAcl +func TestS3ApiController_PutBucketAcl(t *testing.T) { + invalidBody, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{ + Grants: []auth.Grant{ + { + Permission: auth.Permission("invalid_permission"), + }, + }, + }, + }) + assert.NoError(t, err) + + incorrectOwnerBody, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("user"), + }, + AccessControlList: auth.AccessControlList{}, + }) + assert.NoError(t, err) + + validAccessControlPolicy, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{}, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "access denied", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "fails to get bucket ownership", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrInternalError), + extraMockResp: types.ObjectOwnership(""), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInternalError), + }, + }, + { + name: "acl not supported", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerEnforced, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAclNotSupported), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedACL), + }, + }, + { + name: "invalid access control policy", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: invalidBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedACL), + }, + }, + { + name: "incorrect owner id", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: incorrectOwnerBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.APIError{ + Code: "InvalidArgument", + Description: "Invalid id", + HTTPStatusCode: http.StatusBadRequest, + }, + }, + }, + { + name: "both access control policy and grants", + input: testInput{ + body: validAccessControlPolicy, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrUnexpectedContent), + }, + }, + { + name: "access control policy success", + input: testInput{ + body: validAccessControlPolicy, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "invalid canned acl", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "invalid_acl", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "both canned acl and grants", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + "X-Amz-Grant-Read": "grt1", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), + }, + }, + { + name: "canned acl success", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "grants update acl fails", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Grant-Read": "grt1", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: errors.New("accounts does not exist: grt1"), + }, + }, + { + name: "no option provided", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMissingSecurityHeader), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error { + return tt.input.beErr + }, + GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { + return tt.input.extraMockResp.(types.ObjectOwnership), tt.input.extraMockErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + iam: auth.NewIAMServiceSingle( + auth.Account{ + Access: "root", + }), + } + + testController(t, ctrl.PutBucketAcl, tt.output.response, tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + bucket: tt.input.bucket, + headers: tt.input.headers, + }) + }) + } +} diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index b86bffc..9fd7104 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -461,7 +461,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { *res.ContentLength) return &Response{ MetaOpts: &MetaOptions{ - ContentLength: getint64(res.ContentLength), + ContentLength: utils.GetInt64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, }, @@ -501,7 +501,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), }, MetaOpts: &MetaOptions{ - ContentLength: getint64(res.ContentLength), + ContentLength: utils.GetInt64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, }, diff --git a/s3api/router.go b/s3api/router.go index 567ab62..2435642 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -29,7 +29,7 @@ type S3ApiRouter struct { WithAdmSrv bool } -func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool, region string, root middlewares.RootUserConfig) { +func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm metrics.Manager, debug bool, readonly bool, region string, root middlewares.RootUserConfig) { ctrl := controllers.New(be, iam, logger, evs, mm, debug, readonly) adminServices := &controllers.Services{ Logger: aLogger, diff --git a/s3api/server.go b/s3api/server.go index 857bf81..c25d265 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -51,7 +51,7 @@ func New( l s3log.AuditLogger, adminLogger s3log.AuditLogger, evs s3event.S3EventSender, - mm *metrics.Manager, + mm metrics.Manager, opts ...Option, ) (*S3ApiServer, error) { server := &S3ApiServer{ diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 5489d9c..c6f0535 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -704,3 +704,12 @@ func FormatDatePtrToString(date *time.Time, format string) *string { formatted := date.UTC().Format(format) return &formatted } + +// GetInt64 returns the value of int64 pointer +func GetInt64(n *int64) int64 { + if n == nil { + return 0 + } + + return *n +}