From 39cef57c87ba2e3895c4e6101608d292d493b219 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 29 May 2025 00:13:20 +0400 Subject: [PATCH 01/20] feat: implements public bucket access. This implementation introduces **public buckets**, which are accessible without signature-based authentication. There are two ways to grant public access to a bucket: * **Bucket ACLs** * **Bucket Policies** Only `Get` and `List` operations are permitted on public buckets. All **write operations** require authentication, regardless of whether public access is granted through an ACL or a policy. The implementation includes an `AuthorizePublicBucketAccess` middleware, which checks if public access has been granted to the bucket. If so, authentication middlewares are skipped. For unauthenticated requests, appropriate errors are returned based on the specific S3 action. --- **1. Bucket-Level Operations:** ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::test" } ] } ``` **2. Object-Level Operations:** ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::test/*" } ] } ``` **3. Both Bucket and Object-Level Operations:** ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::test" }, { "Effect": "Allow", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::test/*" } ] } ``` --- ```sh aws s3api create-bucket --bucket test --object-ownership BucketOwnerPreferred aws s3api put-bucket-acl --bucket test --acl public-read ``` --- s3api/middlewares/public-bucket.go | 260 +++----------------------- s3api/middlewares/set-defualt-keys.go | 37 ++++ 2 files changed, 65 insertions(+), 232 deletions(-) create mode 100644 s3api/middlewares/set-defualt-keys.go diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index 567324ff..ebf72304 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -24,26 +24,40 @@ import ( "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { +// AuthorizePublicBucketAccess checks if the bucket grants public +// access to anonymous requesters +func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPermission auth.Action, permission auth.Permission) fiber.Handler { return func(ctx *fiber.Ctx) error { // skip for auhtneicated requests if ctx.Query("X-Amz-Algorithm") != "" || ctx.Get("Authorization") != "" { - return ctx.Next() + return nil + } + + switch s3action { + case metrics.ActionListAllMyBuckets: + return s3err.GetAPIError(s3err.ErrAccessDenied) + case metrics.ActionGetBucketOwnershipControls: + return s3err.GetAPIError(s3err.ErrAnonymousGetBucketOwnership) + case metrics.ActionPutBucketOwnershipControls, metrics.ActionDeleteBucketOwnershipControls: + return s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership) + case metrics.ActionPutBucketAcl, metrics.ActionPutObjectAcl, metrics.ActionSelectObjectContent, metrics.ActionCreateBucket: + return s3err.GetAPIError(s3err.ErrAnonymousRequest) + case metrics.ActionCopyObject: + return s3err.GetAPIError(s3err.ErrAnonymousCopyObject) + case metrics.ActionCreateMultipartUpload: + return s3err.GetAPIError(s3err.ErrAnonymousCreateMp) + case metrics.ActionUploadPartCopy, metrics.ActionDeleteObjects: + // TODO: should be fixed with https://github.com/versity/versitygw/issues/1327 + // TODO: should be fixed with https://github.com/versity/versitygw/issues/1338 + return s3err.GetAPIError(s3err.ErrAccessDenied) } bucket, object := parsePath(ctx.Path()) - - action, permission, err := detectS3Action(ctx, object == "") + err := auth.VerifyPublicAccess(ctx.Context(), be, policyPermission, permission, bucket, object) if err != nil { - return sendResponse(ctx, err, l, mm) - } - - err = auth.VerifyPublicAccess(ctx.Context(), be, action, permission, bucket, object) - if err != nil { - return sendResponse(ctx, err, l, mm) + return err } if utils.IsBigDataAction(ctx) { @@ -51,7 +65,7 @@ func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *me if utils.IsUnsignedStreamingPayload(payloadType) { checksumType, err := utils.ExtractChecksumType(ctx) if err != nil { - return sendResponse(ctx, err, l, mm) + return err } wrapBodyReader(ctx, func(r io.Reader) io.Reader { @@ -60,232 +74,14 @@ func AuthorizePublicBucketAccess(be backend.Backend, l s3log.AuditLogger, mm *me return cr }) if err != nil { - return sendResponse(ctx, err, l, mm) + return err } } } utils.ContextKeyPublicBucket.Set(ctx, true) - return ctx.Next() - } -} - -func detectS3Action(ctx *fiber.Ctx, isBucketAction bool) (auth.Action, auth.Permission, error) { - path := ctx.Path() - // ListBuckets is not publically available - if path == "/" { - //TODO: Still not clear what kind of error should be returned in this case(ListBuckets) - return "", auth.PermissionRead, s3err.GetAPIError(s3err.ErrAccessDenied) - } - - queryArgs := ctx.Context().QueryArgs() - - switch ctx.Method() { - case fiber.MethodPatch: - // Admin apis should always be protected - return "", "", s3err.GetAPIError(s3err.ErrAccessDenied) - case fiber.MethodHead: - // HeadBucket - if isBucketAction { - return auth.ListBucketAction, auth.PermissionRead, nil - } - - // HeadObject - return auth.GetObjectAction, auth.PermissionRead, nil - case fiber.MethodGet: - if isBucketAction { - if queryArgs.Has("tagging") { - // GetBucketTagging - return auth.GetBucketTaggingAction, auth.PermissionRead, nil - } else if queryArgs.Has("ownershipControls") { - // GetBucketOwnershipControls - return auth.GetBucketOwnershipControlsAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousGetBucketOwnership) - } else if queryArgs.Has("versioning") { - // GetBucketVersioning - return auth.GetBucketVersioningAction, auth.PermissionRead, nil - } else if queryArgs.Has("policy") { - // GetBucketPolicy - return auth.GetBucketPolicyAction, auth.PermissionRead, nil - } else if queryArgs.Has("cors") { - // GetBucketCors - return auth.GetBucketCorsAction, auth.PermissionRead, nil - } else if queryArgs.Has("versions") { - // ListObjectVersions - return auth.ListBucketVersionsAction, auth.PermissionRead, nil - } else if queryArgs.Has("object-lock") { - // GetObjectLockConfiguration - return auth.GetBucketObjectLockConfigurationAction, auth.PermissionReadAcp, nil - } else if queryArgs.Has("acl") { - // GetBucketAcl - return auth.GetBucketAclAction, auth.PermissionRead, nil - } else if queryArgs.Has("uploads") { - // ListMultipartUploads - return auth.ListBucketMultipartUploadsAction, auth.PermissionRead, nil - } else if queryArgs.GetUintOrZero("list-type") == 2 { - // ListObjectsV2 - return auth.ListBucketAction, auth.PermissionRead, nil - } - // All the other requests are considerd as ListObjects in the router - // no matter what kind of query arguments are provided apart from the ones above - - return auth.ListBucketAction, auth.PermissionRead, nil - } - - if queryArgs.Has("tagging") { - // GetObjectTagging - return auth.GetObjectTaggingAction, auth.PermissionRead, nil - } else if queryArgs.Has("retention") { - // GetObjectRetention - return auth.GetObjectRetentionAction, auth.PermissionRead, nil - } else if queryArgs.Has("legal-hold") { - // GetObjectLegalHold - return auth.GetObjectLegalHoldAction, auth.PermissionReadAcp, nil - } else if queryArgs.Has("acl") { - // GetObjectAcl - return auth.GetObjectAclAction, auth.PermissionRead, nil - } else if queryArgs.Has("attributes") { - // GetObjectAttributes - return auth.GetObjectAttributesAction, auth.PermissionRead, nil - } else if queryArgs.Has("uploadId") { - // ListParts - return auth.ListMultipartUploadPartsAction, auth.PermissionRead, nil - } - - // All the other requests are considerd as GetObject in the router - // no matter what kind of query arguments are provided apart from the ones above - if queryArgs.Has("versionId") { - return auth.GetObjectVersionAction, auth.PermissionRead, nil - } - return auth.GetObjectAction, auth.PermissionRead, nil - case fiber.MethodPut: - if isBucketAction { - if queryArgs.Has("tagging") { - // PutBucketTagging - return auth.PutBucketTaggingAction, auth.PermissionWrite, nil - } - if queryArgs.Has("ownershipControls") { - // PutBucketOwnershipControls - return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership) - } - if queryArgs.Has("versioning") { - // PutBucketVersioning - return auth.PutBucketVersioningAction, auth.PermissionWrite, nil - } - if queryArgs.Has("object-lock") { - // PutObjectLockConfiguration - return auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite, nil - } - if queryArgs.Has("cors") { - // PutBucketCors - return auth.PutBucketCorsAction, auth.PermissionWrite, nil - } - if queryArgs.Has("policy") { - // PutBucketPolicy - return auth.PutBucketPolicyAction, auth.PermissionWrite, nil - } - if queryArgs.Has("acl") { - // PutBucketAcl - return auth.PutBucketAclAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousRequest) - } - - // All the other rquestes are considered as 'CreateBucket' in the router - return "", "", s3err.GetAPIError(s3err.ErrAnonymousRequest) - } - - if queryArgs.Has("tagging") { - // PutObjectTagging - return auth.PutObjectTaggingAction, auth.PermissionWrite, nil - } - if queryArgs.Has("retention") { - // PutObjectRetention - return auth.PutObjectRetentionAction, auth.PermissionWrite, nil - } - if queryArgs.Has("legal-hold") { - // PutObjectLegalHold - return auth.PutObjectLegalHoldAction, auth.PermissionWrite, nil - } - if queryArgs.Has("acl") { - // PutObjectAcl - return auth.PutObjectAclAction, auth.PermissionWriteAcp, s3err.GetAPIError(s3err.ErrAnonymousRequest) - } - if queryArgs.Has("uploadId") && queryArgs.Has("partNumber") { - if ctx.Get("X-Amz-Copy-Source") != "" { - // UploadPartCopy - //TODO: Add public access check for copy-source - // Return AccessDenied for now - return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied) - } - - utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream()) - // UploadPart - return auth.PutObjectAction, auth.PermissionWrite, nil - } - if ctx.Get("X-Amz-Copy-Source") != "" { - return auth.PutObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousCopyObject) - } - - utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream()) - // All the other requests are considered as 'PutObject' in the router - return auth.PutObjectAction, auth.PermissionWrite, nil - case fiber.MethodPost: - if isBucketAction { - // DeleteObjects - // FIXME: should be fixed with https://github.com/versity/versitygw/issues/1327 - // Return AccessDenied for now - return auth.DeleteObjectAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAccessDenied) - } - - if queryArgs.Has("restore") { - return auth.RestoreObjectAction, auth.PermissionWrite, nil - } - if queryArgs.Has("select") && ctx.Query("select-type") == "2" { - // SelectObjectContent - return auth.GetObjectAction, auth.PermissionRead, s3err.GetAPIError(s3err.ErrAnonymousRequest) - } - if queryArgs.Has("uploadId") { - // CompleteMultipartUpload - return auth.PutObjectAction, auth.PermissionWrite, nil - } - - // All the other requests are considered as 'CreateMultipartUpload' in the router - return "", "", s3err.GetAPIError(s3err.ErrAnonymousCreateMp) - case fiber.MethodDelete: - if isBucketAction { - if queryArgs.Has("tagging") { - // DeleteBucketTagging - return auth.PutBucketTaggingAction, auth.PermissionWrite, nil - } - if queryArgs.Has("ownershipControls") { - // DeleteBucketOwnershipControls - return auth.PutBucketOwnershipControlsAction, auth.PermissionWrite, s3err.GetAPIError(s3err.ErrAnonymousPutBucketOwnership) - } - if queryArgs.Has("policy") { - // DeleteBucketPolicy - return auth.PutBucketPolicyAction, auth.PermissionWrite, nil - } - if queryArgs.Has("cors") { - // DeleteBucketCors - return auth.PutBucketCorsAction, auth.PermissionWrite, nil - } - - // All the other requests are considered as 'DeleteBucket' in the router - return auth.DeleteBucketAction, auth.PermissionWrite, nil - } - - if queryArgs.Has("tagging") { - // DeleteObjectTagging - return auth.PutObjectTaggingAction, auth.PermissionWrite, nil - } - if queryArgs.Has("uploadId") { - // AbortMultipartUpload - return auth.AbortMultipartUploadAction, auth.PermissionWrite, nil - } - // All the other requests are considered as 'DeleteObject' in the router - return auth.DeleteObjectAction, auth.PermissionWrite, nil - default: - // In no action is detected, return AccessDenied ? - return "", "", s3err.GetAPIError(s3err.ErrAccessDenied) + return nil } } diff --git a/s3api/middlewares/set-defualt-keys.go b/s3api/middlewares/set-defualt-keys.go new file mode 100644 index 00000000..e87cbbfe --- /dev/null +++ b/s3api/middlewares/set-defualt-keys.go @@ -0,0 +1,37 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" +) + +func SetDefaultValues(root RootUserConfig, region string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + // These are necessary for the server access logs + utils.ContextKeyRegion.Set(ctx, region) + utils.ContextKeyStartTime.Set(ctx, time.Now()) + utils.ContextKeyRootAccessKey.Set(ctx, root.Access) + // Set the account and isRoot to some defulat values, to avoid panics + // in case of public buckets + utils.ContextKeyAccount.Set(ctx, auth.Account{}) + utils.ContextKeyIsRoot.Set(ctx, false) + return ctx.Next() + } +} From edaf9d6d4ef0e9fb378e0b10b20867a07c14817e Mon Sep 17 00:00:00 2001 From: niksis02 Date: Tue, 3 Jun 2025 02:22:43 +0400 Subject: [PATCH 02/20] feat: implements public bucket access for write operations Public buckets support a set of actions on buckets and objects, returning various errors based on the S3 action type and permissions (ACL or policy). The implementation aligns with the table provided in [this gist](https://gist.github.com/niksis02/5919d52d6112537a31c14d9abfa89ac0). --- s3api/middlewares/set-defualt-keys.go | 37 --------------------------- 1 file changed, 37 deletions(-) delete mode 100644 s3api/middlewares/set-defualt-keys.go diff --git a/s3api/middlewares/set-defualt-keys.go b/s3api/middlewares/set-defualt-keys.go deleted file mode 100644 index e87cbbfe..00000000 --- a/s3api/middlewares/set-defualt-keys.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package middlewares - -import ( - "time" - - "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/s3api/utils" -) - -func SetDefaultValues(root RootUserConfig, region string) fiber.Handler { - return func(ctx *fiber.Ctx) error { - // These are necessary for the server access logs - utils.ContextKeyRegion.Set(ctx, region) - utils.ContextKeyStartTime.Set(ctx, time.Now()) - utils.ContextKeyRootAccessKey.Set(ctx, root.Access) - // Set the account and isRoot to some defulat values, to avoid panics - // in case of public buckets - utils.ContextKeyAccount.Set(ctx, auth.Account{}) - utils.ContextKeyIsRoot.Set(ctx, false) - return ctx.Next() - } -} From f877502ab0508ea23e3849a981b7e7591e64464a Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 11 Jun 2025 00:55:58 +0400 Subject: [PATCH 03/20] feat: adds integration tests for public buckets. --- s3api/controllers/base.go | 6 +++--- s3api/middlewares/public-bucket.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 2b8bb9c5..e4ecbaed 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -15,7 +15,6 @@ package controllers import ( - "bytes" "encoding/xml" "errors" "fmt" @@ -2186,7 +2185,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { if bodyi != nil { body = bodyi.(io.Reader) } else { - body = bytes.NewReader([]byte{}) + body = ctx.Request().BodyStream() } res, err := c.be.UploadPart(ctx.Context(), @@ -2659,7 +2658,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { if bodyi != nil { body = bodyi.(io.Reader) } else { - body = bytes.NewReader([]byte{}) + body = ctx.Request().BodyStream() } res, err := c.be.PutObject(ctx.Context(), @@ -3924,6 +3923,7 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { ctx.Status(apierr.HTTPStatusCode) return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", "")) } + fmt.Println(err, "------------") fmt.Fprintf(os.Stderr, "Internal Error, %v\n", err) ctx.Status(http.StatusInternalServerError) diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index ebf72304..e32e7557 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -79,6 +79,25 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm } } + if utils.IsBigDataAction(ctx) { + payloadType := ctx.Get("X-Amz-Content-Sha256") + if utils.IsUnsignedStreamingPayload(payloadType) { + checksumType, err := utils.ExtractChecksumType(ctx) + if err != nil { + return sendResponse(ctx, err, l, mm) + } + + wrapBodyReader(ctx, func(r io.Reader) io.Reader { + var cr io.Reader + cr, err = utils.NewUnsignedChunkReader(r, checksumType) + return cr + }) + if err != nil { + return sendResponse(ctx, err, l, mm) + } + } + } + utils.ContextKeyPublicBucket.Set(ctx, true) return nil From b8456bc5abbec9377b9be812a03c9e2222218f16 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 12 Jun 2025 23:31:41 +0400 Subject: [PATCH 04/20] feat: implements advanced routing system for the bucket get operations. Closes #908 This PR introduces a new routing system integrated with Fiber. It matches each S3 action to a route using middleware utility functions (e.g., URL query match, request header match). Each S3 action is mapped to a dedicated route in the Fiber router. This functionality cannot be achieved using standard Fiber methods, as Fiber lacks the necessary tooling for such dynamic routing. Additionally, this PR implements a generic response handler to manage responses from the backend. This abstraction helps isolate the controller from the data layer and Fiber-specific response logic. With this approach, controller unit testing becomes simpler and more effective. --- s3api/controllers/base.go | 511 ---------------------- s3api/controllers/base_test.go | 209 --------- s3api/controllers/bucket-get.go | 587 ++++++++++++++++++++++++++ s3api/controllers/utilities.go | 181 ++++++++ s3api/middlewares/router-utilities.go | 49 +++ s3api/router.go | 16 +- s3api/utils/context-keys.go | 5 + 7 files changed, 833 insertions(+), 725 deletions(-) create mode 100644 s3api/controllers/bucket-get.go create mode 100644 s3api/controllers/utilities.go create mode 100644 s3api/middlewares/router-utilities.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index e4ecbaed..6f6861c6 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -725,509 +725,6 @@ func getint64(i *int64) int64 { return *i } -func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - prefix := ctx.Query("prefix") - cToken := ctx.Query("continuation-token") - sAfter := ctx.Query("start-after") - marker := ctx.Query("marker") - delimiter := ctx.Query("delimiter") - maxkeysStr := ctx.Query("max-keys") - keyMarker := ctx.Query("key-marker") - maxUploadsStr := ctx.Query("max-uploads") - uploadIdMarker := ctx.Query("upload-id-marker") - versionIdMarker := ctx.Query("version-id-marker") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - - if ctx.Request().URI().QueryArgs().Has("tagging") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketTaggingAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - tags, err := c.be.GetBucketTagging(ctx.Context(), bucket) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - resp := s3response.Tagging{ - TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, - } - - for key, val := range tags { - resp.TagSet.Tags = append(resp.TagSet.Tags, - s3response.Tag{Key: key, Value: val}) - } - - return SendXMLResponse(ctx, resp, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("ownershipControls") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketOwnershipControlsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) - return SendXMLResponse(ctx, - s3response.OwnershipControls{ - Rules: []types.OwnershipControlsRule{ - { - ObjectOwnership: data, - }, - }, - }, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("versioning") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketVersioningAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - // Only admin users and the bucket owner are allowed to get the versioning state of a bucket. - if err := auth.IsAdminOrOwner(acct, isRoot, parsedAcl); err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetBucketVersioning(ctx.Context(), bucket) - return SendXMLResponse(ctx, data, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("policy") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketPolicyAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetBucketPolicy(ctx.Context(), bucket) - return SendXMLResponse(ctx, data, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("cors") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketCorsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetBucketCors(ctx.Context(), bucket) - return SendXMLResponse(ctx, data, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("versions") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketVersionsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectVersions, - BucketOwner: parsedAcl.Owner, - }) - } - - maxkeys, err := utils.ParseUint(maxkeysStr) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max keys %q: %v", - maxkeysStr, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectVersions, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.ListObjectVersions(ctx.Context(), - &s3.ListObjectVersionsInput{ - Bucket: &bucket, - Delimiter: &delimiter, - KeyMarker: &keyMarker, - MaxKeys: &maxkeys, - Prefix: &prefix, - VersionIdMarker: &versionIdMarker, - }) - return SendXMLResponse(ctx, data, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectVersions, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("object-lock") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketObjectLockConfigurationAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetObjectLockConfiguration(ctx.Context(), bucket) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - resp, err := auth.ParseBucketLockConfigurationOutput(data) - return SendXMLResponse(ctx, resp, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("acl") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionReadAcp, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.GetBucketAclAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetBucketAcl(ctx.Context(), - &s3.GetBucketAclInput{Bucket: &bucket}) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - }) - } - - res, err := auth.ParseACLOutput(data, parsedAcl.Owner) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("uploads") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketMultipartUploadsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListMultipartUploads, - BucketOwner: parsedAcl.Owner, - }) - } - maxUploads, err := utils.ParseUint(maxUploadsStr) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max uploads %q: %v", - maxUploadsStr, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxUploads), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListMultipartUploads, - BucketOwner: parsedAcl.Owner, - }) - } - res, err := c.be.ListMultipartUploads(ctx.Context(), - &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - Delimiter: &delimiter, - Prefix: &prefix, - UploadIdMarker: &uploadIdMarker, - MaxUploads: &maxUploads, - KeyMarker: &keyMarker, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListMultipartUploads, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.QueryInt("list-type") == 2 { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectsV2, - BucketOwner: parsedAcl.Owner, - }) - } - maxkeys, err := utils.ParseUint(maxkeysStr) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max keys %q: %v", - maxkeysStr, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectsV2, - BucketOwner: parsedAcl.Owner, - }) - } - - fetchOwner := strings.EqualFold(ctx.Query("fetch-owner"), "true") - res, err := c.be.ListObjectsV2(ctx.Context(), - &s3.ListObjectsV2Input{ - Bucket: &bucket, - Prefix: &prefix, - ContinuationToken: &cToken, - Delimiter: &delimiter, - MaxKeys: &maxkeys, - StartAfter: &sAfter, - FetchOwner: &fetchOwner, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjectsV2, - BucketOwner: parsedAcl.Owner, - }) - } - - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - maxkeys, err := utils.ParseUint(maxkeysStr) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max keys %q: %v", - maxkeysStr, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxKeys), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.ListObjects(ctx.Context(), - &s3.ListObjectsInput{ - Bucket: &bucket, - Prefix: &prefix, - Marker: &marker, - Delimiter: &delimiter, - MaxKeys: &maxkeys, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListObjects, - BucketOwner: parsedAcl.Owner, - }) -} - func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") acl := ctx.Get("X-Amz-Acl") @@ -3949,14 +3446,6 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { return nil } -var ( - xmlhdr = []byte(`` + "\n") -) - -const ( - maxXMLBodyLen = 4 * 1024 * 1024 -) - func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error { if l.MetricsMng != nil { if l.ObjectCount > 0 { diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index f9d83d54..25e57bea 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -364,215 +364,6 @@ func TestS3ApiController_GetActions(t *testing.T) { } } -func TestS3ApiController_ListActions(t *testing.T) { - type args struct { - req *http.Request - } - - objectLockResult, err := json.Marshal(auth.BucketLockConfig{}) - if err != nil { - t.Errorf("failed to parse object lock result %v", err) - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { - return s3response.ListMultipartUploadsResult{}, nil - }, - ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { - return s3response.ListObjectsV2Result{}, nil - }, - ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { - return s3response.ListObjectsResult{}, nil - }, - GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { - return map[string]string{}, nil - }, - GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { - return s3response.GetBucketVersioningOutput{}, nil - }, - ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { - return s3response.ListVersionsResult{}, nil - }, - GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return []byte{}, nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return objectLockResult, nil - }, - GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { - return types.ObjectOwnershipBucketOwnerEnforced, nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - - app.Get("/:bucket", s3ApiController.ListActions) - - // Error case - s3ApiControllerError := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { - return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) - }, - GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) - }, - }, - } - appError := fiber.New() - appError.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - appError.Get("/:bucket", s3ApiControllerError.ListActions) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Get-bucket-tagging-non-existing-bucket", - app: appError, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil), - }, - wantErr: false, - statusCode: 404, - }, - { - name: "Get-bucket-ownership-control-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-bucket-tagging-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-object-lock-configuration-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-bucket-acl-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?acl=acl", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-Multipart-Upload-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?uploads=uploads", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-Objects-V2-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?list-type=2", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-Objects-V1-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-actions-error-case", - app: appError, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 501, - }, - { - name: "List-actions-get-bucket-versioning-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?versioning", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-actions-get-bucket-policy-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?policy", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "List-actions-list-object-versions-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket?versions", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.ListActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.ListActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - }) - } -} - func TestS3ApiController_PutBucketActions(t *testing.T) { type args struct { req *http.Request diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go new file mode 100644 index 00000000..d115519b --- /dev/null +++ b/s3api/controllers/bucket-get.go @@ -0,0 +1,587 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) GetBucketTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketTaggingAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + tags, err := c.be.GetBucketTagging(ctx.Context(), bucket) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + resp := s3response.Tagging{ + TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, + } + + for key, val := range tags { + resp.TagSet.Tags = append(resp.TagSet.Tags, + s3response.Tag{Key: key, Value: val}) + } + + return &Response{ + Data: resp, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetBucketOwnershipControls(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketOwnershipControlsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) + return &Response{ + Data: s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{ + { + ObjectOwnership: data, + }, + }, + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetBucketVersioning(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketVersioningAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + // Only admin users and the bucket owner are allowed to get the versioning state of a bucket. + if err := auth.IsAdminOrOwner(acct, isRoot, parsedAcl); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketVersioning(ctx.Context(), bucket) + return &Response{ + Data: data, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetBucketCors(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketCorsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketCors(ctx.Context(), bucket) + return &Response{ + Data: data, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetBucketPolicy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketPolicyAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketPolicy(ctx.Context(), bucket) + return &Response{ + Data: data, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListObjectVersions(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + prefix := ctx.Query("prefix") + delimiter := ctx.Query("delimiter") + maxkeysStr := ctx.Query("max-keys") + keyMarker := ctx.Query("key-marker") + versionIdMarker := ctx.Query("version-id-marker") + // context keys + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.ListBucketVersionsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectVersions, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + maxkeys, err := utils.ParseUint(maxkeysStr) + if err != nil { + debuglogger.Logf("error parsing max keys %q: %v", + maxkeysStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectVersions, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) + } + + data, err := c.be.ListObjectVersions(ctx.Context(), + &s3.ListObjectVersionsInput{ + Bucket: &bucket, + Delimiter: &delimiter, + KeyMarker: &keyMarker, + MaxKeys: &maxkeys, + Prefix: &prefix, + VersionIdMarker: &versionIdMarker, + }) + return &Response{ + Data: data, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectVersions, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectLockConfiguration(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + // context keys + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketObjectLockConfigurationAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectLockConfiguration(ctx.Context(), bucket) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + resp, err := auth.ParseBucketLockConfigurationOutput(data) + return &Response{ + Data: resp, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetBucketAcl(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + // context keys + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionReadAcp, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.GetBucketAclAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetBucketAcl(ctx.Context(), + &s3.GetBucketAclInput{Bucket: &bucket}) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := auth.ParseACLOutput(data, parsedAcl.Owner) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListMultipartUploads(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + prefix := ctx.Query("prefix") + delimiter := ctx.Query("delimiter") + keyMarker := ctx.Query("key-marker") + maxUploadsStr := ctx.Query("max-uploads") + uploadIdMarker := ctx.Query("upload-id-marker") + // context keys + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.ListBucketMultipartUploadsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListMultipartUploads, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + maxUploads, err := utils.ParseUint(maxUploadsStr) + if err != nil { + debuglogger.Logf("error parsing max uploads %q: %v", + maxUploadsStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListMultipartUploads, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxUploads) + } + res, err := c.be.ListMultipartUploads(ctx.Context(), + &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + Delimiter: &delimiter, + Prefix: &prefix, + UploadIdMarker: &uploadIdMarker, + MaxUploads: &maxUploads, + KeyMarker: &keyMarker, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListMultipartUploads, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListObjectsV2(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + prefix := ctx.Query("prefix") + cToken := ctx.Query("continuation-token") + sAfter := ctx.Query("start-after") + delimiter := ctx.Query("delimiter") + maxkeysStr := ctx.Query("max-keys") + fetchOwner := strings.EqualFold(ctx.Query("fetch-owner"), "true") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.ListBucketAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectsV2, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + maxkeys, err := utils.ParseUint(maxkeysStr) + if err != nil { + debuglogger.Logf("error parsing max keys %q: %v", + maxkeysStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectsV2, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) + } + + res, err := c.be.ListObjectsV2(ctx.Context(), + &s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: &prefix, + ContinuationToken: &cToken, + Delimiter: &delimiter, + MaxKeys: &maxkeys, + StartAfter: &sAfter, + FetchOwner: &fetchOwner, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjectsV2, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListObjects(ctx *fiber.Ctx) (*Response, error) { + // url values + bucket := ctx.Params("bucket") + prefix := ctx.Query("prefix") + marker := ctx.Query("marker") + delimiter := ctx.Query("delimiter") + maxkeysStr := ctx.Query("max-keys") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.ListBucketAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + maxkeys, err := utils.ParseUint(maxkeysStr) + if err != nil { + debuglogger.Logf("error parsing max keys %q: %v", + maxkeysStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjects, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) + } + + res, err := c.be.ListObjects(ctx.Context(), + &s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + Marker: &marker, + Delimiter: &delimiter, + MaxKeys: &maxkeys, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go new file mode 100644 index 00000000..97d66a70 --- /dev/null +++ b/s3api/controllers/utilities.go @@ -0,0 +1,181 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "fmt" + "net/http" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3log" +) + +var ( + xmlhdr = []byte(`` + "\n") +) + +const ( + maxXMLBodyLen = 4 * 1024 * 1024 +) + +type MetaOptions struct { + ContentLength int64 + Action string + BucketOwner string + ObjectSize int64 + ObjectCount int64 + EventName s3event.EventType + ObjectETag *string + VersionId *string + Status int +} + +type Response struct { + Data any + Headers map[string]string + MetaOpts *MetaOptions +} + +type Handler func(ctx *fiber.Ctx) (*Response, error) + +func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event.S3EventSender, mm *metrics.Manager) fiber.Handler { + return func(ctx *fiber.Ctx) error { + // if skip locals is set, skip to the next rout handler + if utils.ContextKeySkip.IsSet(ctx) { + utils.ContextKeySkip.Delete(ctx) + return ctx.Next() + } + + response, err := handler(ctx) + opts := response.MetaOpts + // Send the metrics + if mm != nil { + if opts.ObjectCount > 0 { + mm.Send(ctx, err, opts.Action, opts.ObjectCount, opts.Status) + } else { + mm.Send(ctx, err, opts.Action, opts.ContentLength, opts.Status) + } + } + // Handle the error case + if err != nil { + // Audit the error log + if s3logger != nil { + s3logger.Log(ctx, err, nil, s3log.LogMeta{ + Action: opts.Action, + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, + }) + } + serr, ok := err.(s3err.APIError) + if ok { + ctx.Status(serr.HTTPStatusCode) + return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", "")) + } + + debuglogger.Logf("Internal Error, %v", err) + ctx.Status(http.StatusInternalServerError) + + // If the error is not 's3err.APIError' return 'InternalError' + return ctx.Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) + } + + // Set the response headers + SetResponseHeaders(ctx, response.Headers) + + if opts.Status == 0 { + opts.Status = http.StatusOK + } + + // if no data payload is provided, send the response status + if response.Data == nil { + ctx.Status(opts.Status) + return nil + } + + var responseBytes []byte + + // Handle already encoded responses(text, json...) + encodedResp, ok := response.Data.([]byte) + if ok { + responseBytes = encodedResp + } else { + if responseBytes, err = xml.Marshal(response.Data); err != nil { + return err + } + + if len(responseBytes) > 0 { + ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) + ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) + } + } + + if s3logger != nil { + s3logger.Log(ctx, nil, responseBytes, s3log.LogMeta{ + Action: opts.Action, + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, + }) + } + + if s3evnt != nil { + s3evnt.SendEvent(ctx, s3event.EventMeta{ + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, + ObjectETag: opts.ObjectETag, + VersionId: opts.VersionId, + EventName: opts.EventName, + }) + } + + if ok { + if len(responseBytes) > 0 { + ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) + } + + return ctx.Send(responseBytes) + } + + msglen := len(xmlhdr) + len(responseBytes) + if msglen > maxXMLBodyLen { + debuglogger.Logf("XML encoded body len %v exceeds max len %v", + msglen, maxXMLBodyLen) + ctx.Status(http.StatusInternalServerError) + + return ctx.Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) + } + res := make([]byte, 0, msglen) + res = append(res, xmlhdr...) + res = append(res, responseBytes...) + + return ctx.Send(res) + } +} + +func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]string) { + if headers == nil { + return + } + for key, val := range headers { + ctx.Response().Header.Add(key, val) + } +} diff --git a/s3api/middlewares/router-utilities.go b/s3api/middlewares/router-utilities.go new file mode 100644 index 00000000..2a09f251 --- /dev/null +++ b/s3api/middlewares/router-utilities.go @@ -0,0 +1,49 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/s3api/utils" +) + +func MatchQueryArgs(args ...string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + if utils.ContextKeySkip.IsSet(ctx) { + return ctx.Next() + } + for _, query := range args { + if !ctx.Request().URI().QueryArgs().Has(query) { + utils.ContextKeySkip.Set(ctx, true) + break + } + } + return ctx.Next() + } +} + +func MatchQueryArgWithValue(key, val string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + if utils.ContextKeySkip.IsSet(ctx) { + return ctx.Next() + } + + if ctx.Query(key) != val { + utils.ContextKeySkip.Set(ctx, true) + } + + return ctx.Next() + } +} diff --git a/s3api/router.go b/s3api/router.go index f640cb67..0a60029a 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -67,11 +67,17 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ // HeadBucket app.Head("/:bucket", s3ApiController.HeadBucket) - // GetBucketAcl action - // ListMultipartUploads action - // ListObjects action - // ListObjectsV2 action - app.Get("/:bucket", s3ApiController.ListActions) + app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(s3ApiController.GetBucketTagging, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(s3ApiController.GetBucketOwnershipControls, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(s3ApiController.GetBucketVersioning, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(s3ApiController.GetBucketPolicy, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(s3ApiController.GetBucketCors, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(s3ApiController.GetObjectLockConfiguration, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(s3ApiController.GetBucketAcl, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(s3ApiController.ListMultipartUploads, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(s3ApiController.ListObjectVersions, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(s3ApiController.ListObjectsV2, logger, evs, mm)) + app.Get("/:bucket", controllers.ProcessResponse(s3ApiController.ListObjects, logger, evs, mm)) // HeadObject action app.Head("/:bucket/:key/*", s3ApiController.HeadObject) diff --git a/s3api/utils/context-keys.go b/s3api/utils/context-keys.go index 313248aa..2157240b 100644 --- a/s3api/utils/context-keys.go +++ b/s3api/utils/context-keys.go @@ -34,6 +34,7 @@ const ( ContextKeyParsedAcl ContextKey = "parsed-acl" ContextKeySkipResBodyLog ContextKey = "skip-res-body-log" ContextKeyBodyReader ContextKey = "body-reader" + ContextKeySkip ContextKey = "__skip" ) func (ck ContextKey) Values() []ContextKey { @@ -60,6 +61,10 @@ func (ck ContextKey) IsSet(ctx *fiber.Ctx) bool { return val != nil } +func (ck ContextKey) Delete(ctx *fiber.Ctx) { + ctx.Locals(string(ck), nil) +} + func (ck ContextKey) Get(ctx *fiber.Ctx) any { return ctx.Locals(string(ck)) } From a7c3cb5cf80c6b54de063709fd7670ac5028f6f0 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Fri, 13 Jun 2025 18:22:59 +0400 Subject: [PATCH 05/20] feat: implements advanced routing for ListBuckets, HeadBucket and bucket delete operations --- s3api/controllers/base.go | 209 +----------------------- s3api/controllers/base_test.go | 254 ----------------------------- s3api/controllers/bucket-delete.go | 204 +++++++++++++++++++++++ s3api/controllers/bucket-head.go | 77 +++++++++ s3api/controllers/bucket-list.go | 50 ++++++ s3api/router.go | 57 ++++--- 6 files changed, 362 insertions(+), 489 deletions(-) create mode 100644 s3api/controllers/bucket-delete.go create mode 100644 s3api/controllers/bucket-head.go create mode 100644 s3api/controllers/bucket-list.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 6f6861c6..b771fc50 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -73,45 +73,6 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } -func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error { - cToken := ctx.Query("continuation-token") - prefix := ctx.Query("prefix") - maxBucketsStr := ctx.Query("max-buckets") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - - var maxBuckets int32 = 10000 - if maxBucketsStr != "" { - maxBucketsParsed, err := strconv.ParseInt(maxBucketsStr, 10, 32) - if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > 10000 { - if c.debug { - debuglogger.Logf("error parsing max-buckets %q: %v", maxBucketsStr, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListAllMyBuckets, - }) - } - maxBuckets = int32(maxBucketsParsed) - } - - res, err := c.be.ListBuckets(ctx.Context(), - s3response.ListBucketsInput{ - Owner: acct.Access, - IsAdmin: acct.Role == auth.RoleAdmin, - MaxBuckets: int32(maxBuckets), - ContinuationToken: cToken, - Prefix: prefix, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListAllMyBuckets, - }) -} - func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") key := ctx.Params("key") @@ -2258,174 +2219,6 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { }) } -func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - - if ctx.Request().URI().QueryArgs().Has("tagging") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketTaggingAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteBucketTagging(ctx.Context(), bucket) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketTagging, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - if ctx.Request().URI().QueryArgs().Has("ownershipControls") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketOwnershipControlsAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteBucketOwnershipControls(ctx.Context(), bucket) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - if ctx.Request().URI().QueryArgs().Has("policy") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.DeleteBucketPolicyAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteBucketPolicy(ctx.Context(), bucket) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketPolicy, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - if ctx.Request().URI().QueryArgs().Has("cors") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketCorsAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteBucketCors(ctx.Context(), bucket) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.DeleteBucketAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucket, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteBucket(ctx.Context(), bucket) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteBucket, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) -} - func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) @@ -2688,7 +2481,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { }) } -func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { +func (c S3ApiController) HeadBuckets(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 25e57bea..d85fbc93 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -85,83 +85,6 @@ func TestNew(t *testing.T) { } } -func TestS3ApiController_ListBuckets(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { - return s3response.ListAllMyBucketsResult{}, nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"}) - return ctx.Next() - }) - app.Get("/", s3ApiController.ListBuckets) - - // Error case - appErr := fiber.New() - s3ApiControllerErr := S3ApiController{ - be: &BackendMock{ - ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { - return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) - }, - }, - } - - appErr.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access", Role: "admin:"}) - return ctx.Next() - }) - appErr.Get("/", s3ApiControllerErr.ListBuckets) - - tests := []struct { - name string - args args - app *fiber.App - wantErr bool - statusCode int - }{ - { - name: "List-bucket-method-not-allowed", - args: args{ - req: httptest.NewRequest(http.MethodGet, "/", nil), - }, - app: appErr, - wantErr: false, - statusCode: 405, - }, - { - name: "list-bucket-success", - args: args{ - req: httptest.NewRequest(http.MethodGet, "/", nil), - }, - app: app, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - }) - } -} - func getPtr(val string) *string { return &val } @@ -1044,94 +967,6 @@ func TestS3ApiController_PutActions(t *testing.T) { } } -func TestS3ApiController_DeleteBucket(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - DeleteBucketFunc: func(_ context.Context, bucket string) error { - return nil - }, - DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error { - return nil - }, - DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error { - return nil - }, - DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error { - return nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - - app.Delete("/:bucket", s3ApiController.DeleteBucket) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Delete-bucket-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Delete-bucket-tagging-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket?tagging", nil), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Delete-bucket-ownership-controls-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil), - }, - wantErr: false, - statusCode: 204, - }, { - name: "Delete-bucket-policy-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil), - }, - wantErr: false, - statusCode: 204, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.DeleteBucket() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.DeleteBucket() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - func TestS3ApiController_DeleteObjects(t *testing.T) { type args struct { req *http.Request @@ -1319,95 +1154,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) { } } -func TestS3ApiController_HeadBucket(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { - return &s3.HeadBucketOutput{}, nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - utils.ContextKeyRegion.Set(ctx, "us-east-1") - return ctx.Next() - }) - - app.Head("/:bucket", s3ApiController.HeadBucket) - - // Error case - appErr := fiber.New() - - s3ApiControllerErr := S3ApiController{be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadBucketFunc: func(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrBucketNotEmpty) - }, - }, - } - - appErr.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - utils.ContextKeyRegion.Set(ctx, "us-east-1") - return ctx.Next() - }) - - appErr.Head("/:bucket", s3ApiControllerErr.HeadBucket) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Head-bucket-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Head-bucket-error", - app: appErr, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 409, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.HeadBucket() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.HeadBucket() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - func TestS3ApiController_HeadObject(t *testing.T) { type args struct { req *http.Request diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go new file mode 100644 index 00000000..59a49f62 --- /dev/null +++ b/s3api/controllers/bucket-delete.go @@ -0,0 +1,204 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "net/http" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/utils" +) + +func (c S3ApiController) DeleteBucketTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketTaggingAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucketTagging(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketTagging, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) DeleteBucketOwnershipControls(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketOwnershipControlsAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucketOwnershipControls(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) DeleteBucketPolicy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.DeleteBucketPolicyAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucketPolicy(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketPolicy, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) DeleteBucketCors(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketCorsAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucketCors(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.DeleteBucketAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucket, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteBucket(ctx.Context(), bucket) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteBucket, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} diff --git a/s3api/controllers/bucket-head.go b/s3api/controllers/bucket-head.go new file mode 100644 index 00000000..322d01e4 --- /dev/null +++ b/s3api/controllers/bucket-head.go @@ -0,0 +1,77 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/utils" +) + +func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + region := utils.ContextKeyRegion.Get(ctx).(string) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.ListBucketAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadBucket, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + _, err = c.be.HeadBucket(ctx.Context(), + &s3.HeadBucketInput{ + Bucket: &bucket, + }) + + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadBucket, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + return &Response{ + Headers: map[string]string{ + "X-Amz-Access-Point-Alias": "false", + "X-Amz-Bucket-Region": region, + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadBucket, + BucketOwner: parsedAcl.Owner, + }, + }, nil +} diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go new file mode 100644 index 00000000..9dd649b4 --- /dev/null +++ b/s3api/controllers/bucket-list.go @@ -0,0 +1,50 @@ +package controllers + +import ( + "strconv" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { + cToken := ctx.Query("continuation-token") + prefix := ctx.Query("prefix") + maxBucketsStr := ctx.Query("max-buckets") + + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + + var maxBuckets int32 = 10000 + if maxBucketsStr != "" { + maxBucketsParsed, err := strconv.ParseInt(maxBucketsStr, 10, 32) + if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > 10000 { + debuglogger.Logf("error parsing max-buckets %q: %v", maxBucketsStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListAllMyBuckets, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets) + } + maxBuckets = int32(maxBucketsParsed) + } + + res, err := c.be.ListBuckets(ctx.Context(), + s3response.ListBucketsInput{ + Owner: acct.Access, + IsAdmin: acct.Role == auth.RoleAdmin, + MaxBuckets: int32(maxBuckets), + ContinuationToken: cToken, + Prefix: prefix, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListAllMyBuckets, + }, + }, err +} diff --git a/s3api/router.go b/s3api/router.go index 0a60029a..e6150dd6 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -30,7 +30,7 @@ type S3ApiRouter struct { } func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) { - s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly) + ctrl := controllers.New(be, iam, logger, evs, mm, debug, readonly) if sa.WithAdmSrv { adminController := controllers.NewAdminController(iam, be, aLogger) @@ -54,33 +54,36 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets) } - // ListBuckets action - app.Get("/", s3ApiController.ListBuckets) + app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) - // CreateBucket action - // PutBucketAcl action - app.Put("/:bucket", s3ApiController.PutBucketActions) - - // DeleteBucket action - app.Delete("/:bucket", s3ApiController.DeleteBucket) + // Put bucket operations + app.Put("/:bucket", ctrl.PutBucketActions) // HeadBucket - app.Head("/:bucket", s3ApiController.HeadBucket) + app.Head("/:bucket", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(s3ApiController.GetBucketTagging, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(s3ApiController.GetBucketOwnershipControls, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(s3ApiController.GetBucketVersioning, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(s3ApiController.GetBucketPolicy, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(s3ApiController.GetBucketCors, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(s3ApiController.GetObjectLockConfiguration, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(s3ApiController.GetBucketAcl, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(s3ApiController.ListMultipartUploads, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(s3ApiController.ListObjectVersions, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(s3ApiController.ListObjectsV2, logger, evs, mm)) - app.Get("/:bucket", controllers.ProcessResponse(s3ApiController.ListObjects, logger, evs, mm)) + // Delete bucket operations + app.Delete("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteBucketTagging, logger, evs, mm)) + app.Delete("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.DeleteBucketOwnershipControls, logger, evs, mm)) + app.Delete("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.DeleteBucketPolicy, logger, evs, mm)) + app.Delete("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm)) + app.Delete("/:bucket", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm)) + + // Get bucket operations + app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.GetBucketPolicy, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.GetBucketCors, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.GetObjectLockConfiguration, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetBucketAcl, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.ListMultipartUploads, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(ctrl.ListObjectVersions, logger, evs, mm)) + app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) + app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) // HeadObject action - app.Head("/:bucket/:key/*", s3ApiController.HeadObject) + app.Head("/:bucket/:key/*", ctrl.HeadObject) // GetObjectAcl action // GetObject action @@ -88,21 +91,21 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ // GetObjectTagging action // ListParts action // GetObjectAttributes action - app.Get("/:bucket/:key/*", s3ApiController.GetActions) + app.Get("/:bucket/:key/*", ctrl.GetActions) // DeleteObject action // AbortMultipartUpload action // DeleteObjectTagging action - app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions) + app.Delete("/:bucket/:key/*", ctrl.DeleteActions) // DeleteObjects action - app.Post("/:bucket", s3ApiController.DeleteObjects) + app.Post("/:bucket", ctrl.DeleteObjects) // CompleteMultipartUpload action // CreateMultipartUpload // RestoreObject action // SelectObjectContent action - app.Post("/:bucket/:key/*", s3ApiController.CreateActions) + app.Post("/:bucket/:key/*", ctrl.CreateActions) // CopyObject action // PutObject action @@ -110,5 +113,5 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ // UploadPartCopy action // PutObjectTagging action // PutObjectAcl action - app.Put("/:bucket/:key/*", s3ApiController.PutActions) + app.Put("/:bucket/:key/*", ctrl.PutActions) } From d2038ca9736595964072f4701d2dd89090e5a735 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 16 Jun 2025 18:46:40 +0400 Subject: [PATCH 06/20] feat: implements advanced routing for HeadObject and bucket PUT operations. --- s3api/controllers/base.go | 260 -------------- s3api/controllers/base_test.go | 112 ------ s3api/controllers/bucket-head.go | 6 +- s3api/controllers/bucket-list.go | 14 + s3api/controllers/bucket-put.go | 582 +++++++++++++++++++++++++++++++ s3api/controllers/object-head.go | 160 +++++++++ s3api/controllers/utilities.go | 16 +- s3api/router.go | 13 +- s3api/utils/utils.go | 34 ++ 9 files changed, 813 insertions(+), 384 deletions(-) create mode 100644 s3api/controllers/bucket-put.go create mode 100644 s3api/controllers/object-head.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index b771fc50..3f8ef249 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -2538,266 +2538,6 @@ const ( timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" ) -func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) - versionId := ctx.Query("versionId") - objRange := ctx.Get("Range") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - var partNumber *int32 - if ctx.Request().URI().QueryArgs().Has("partNumber") { - if partNumberQuery < 1 || partNumberQuery > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumberQuery) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - partNumber = &partNumberQuery - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) - if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { - if c.debug { - debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) - } - return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.HeadObject(ctx.Context(), - &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &key, - PartNumber: partNumber, - VersionId: &versionId, - ChecksumMode: checksumMode, - Range: &objRange, - }) - if err != nil { - if res != nil { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-delete-marker", - Value: "true", - }, - { - Key: "Last-Modified", - Value: res.LastModified.UTC().Format(timefmt), - }, - }) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - utils.SetMetaHeaders(ctx, res.Metadata) - headers := []utils.CustomHeader{ - { - Key: "Content-Length", - Value: fmt.Sprint(getint64(res.ContentLength)), - }, - { - Key: "ETag", - Value: getstring(res.ETag), - }, - { - Key: "x-amz-restore", - Value: getstring(res.Restore), - }, - } - if getstring(res.AcceptRanges) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "accept-ranges", - Value: getstring(res.AcceptRanges), - }) - } - if getstring(res.ContentRange) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Range", - Value: getstring(res.ContentRange), - }) - } - if getstring(res.ContentDisposition) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Disposition", - Value: getstring(res.ContentDisposition), - }) - } - if getstring(res.ContentEncoding) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }) - } - if getstring(res.ContentLanguage) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Language", - Value: getstring(res.ContentLanguage), - }) - } - if getstring(res.CacheControl) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Cache-Control", - Value: getstring(res.CacheControl), - }) - } - if getstring(res.ExpiresString) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Expires", - Value: getstring(res.ExpiresString), - }) - } - if res.ObjectLockMode != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-mode", - Value: string(res.ObjectLockMode), - }) - } - if res.ObjectLockLegalHoldStatus != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-legal-hold", - Value: string(res.ObjectLockLegalHoldStatus), - }) - } - if res.ObjectLockRetainUntilDate != nil { - retainUntilDate := res.ObjectLockRetainUntilDate.Format(time.RFC3339) - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-retain-until-date", - Value: retainUntilDate, - }) - } - if res.PartsCount != nil { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-mp-parts-count", - Value: fmt.Sprintf("%v", *res.PartsCount), - }) - } - if res.LastModified != nil { - lastmod := res.LastModified.UTC().Format(timefmt) - headers = append(headers, utils.CustomHeader{ - Key: "Last-Modified", - Value: lastmod, - }) - } - if res.StorageClass != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-storage-class", - Value: string(res.StorageClass), - }) - } - switch { - case res.ChecksumCRC32 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - if res.ChecksumType != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-type", - Value: string(res.ChecksumType), - }) - } - - contentType := getstring(res.ContentType) - if contentType == "" { - contentType = defaultContentType - } - headers = append(headers, utils.CustomHeader{ - Key: "Content-Type", - Value: contentType, - }) - - if getstring(res.VersionId) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - - utils.SetResponseHeaders(ctx, headers) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) -} - func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") key := ctx.Params("key") diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index d85fbc93..d6192783 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -1154,118 +1154,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) { } } -func TestS3ApiController_HeadObject(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - - // Mock values - contentEncoding := "gzip" - contentType := "application/xml" - eTag := "Valid etag" - lastModifie := time.Now() - contentLength := int64(64) - - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { - return &s3.HeadObjectOutput{ - ContentEncoding: &contentEncoding, - ContentLength: &contentLength, - ContentType: &contentType, - LastModified: &lastModifie, - ETag: &eTag, - }, nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Head("/:bucket/:key/*", s3ApiController.HeadObject) - - //Error case - appErr := fiber.New() - - s3ApiControllerErr := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) - }, - }, - } - - appErr.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject) - - invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil) - invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Head-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Head-object-invalid-checksum-mode", - app: app, - args: args{ - req: invChecksumMode, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Head-object-error", - app: appErr, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 400, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.HeadObject() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.HeadObject() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - func TestS3ApiController_CreateActions(t *testing.T) { type args struct { req *http.Request diff --git a/s3api/controllers/bucket-head.go b/s3api/controllers/bucket-head.go index 322d01e4..bac27962 100644 --- a/s3api/controllers/bucket-head.go +++ b/s3api/controllers/bucket-head.go @@ -65,9 +65,9 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { } return &Response{ - Headers: map[string]string{ - "X-Amz-Access-Point-Alias": "false", - "X-Amz-Bucket-Region": region, + Headers: map[string]*string{ + "X-Amz-Access-Point-Alias": utils.GetStringPtr("false"), + "X-Amz-Bucket-Region": utils.GetStringPtr(region), }, MetaOpts: &MetaOptions{ Action: metrics.ActionHeadBucket, diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go index 9dd649b4..c2ceaa9c 100644 --- a/s3api/controllers/bucket-list.go +++ b/s3api/controllers/bucket-list.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package controllers import ( diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go new file mode 100644 index 00000000..5fa73c3f --- /dev/null +++ b/s3api/controllers/bucket-put.go @@ -0,0 +1,582 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) PutBucketTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketTaggingAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitBucket) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketTagging(ctx.Context(), bucket, tagging) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketOwnershipControlsAction, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var ownershipControls s3response.OwnershipControls + if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil { + debuglogger.Logf("failed to unmarshal request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + rulesCount := len(ownershipControls.Rules) + isValidOwnership := utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) + if rulesCount != 1 || !isValidOwnership { + if rulesCount != 1 { + debuglogger.Logf("ownership control rules should be 1, got %v", rulesCount) + } + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketVersioningAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var versioningConf types.VersioningConfiguration + err = xml.Unmarshal(ctx.Body(), &versioningConf) + if err != nil { + debuglogger.Logf("error unmarshalling versioning configuration: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + if versioningConf.Status != types.BucketVersioningStatusEnabled && + versioningConf.Status != types.BucketVersioningStatusSuspended { + debuglogger.Logf("invalid versioning configuration status: %v", versioningConf.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectLockConfiguration(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketObjectLockConfigurationAction, + IsBucketPublic: isPublicBucket, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + config, err := auth.ParseBucketLockConfigurationInput(ctx.Body()) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectLockConfiguration(ctx.Context(), bucket, config) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketCors(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketCorsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketCors(ctx.Context(), []byte{}) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketPolicyAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body()) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + // context locals + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + var input *auth.PutBucketAclInput + + ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + if ownership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled") + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAclNotSupported) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWriteAcp, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketAclAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if len(ctx.Body()) > 0 { + var accessControlPolicy auth.AccessControlPolicy + err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) + if err != nil { + debuglogger.Logf("error unmarshalling access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedACL) + } + + err = accessControlPolicy.Validate() + if err != nil { + debuglogger.Logf("invalid access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if *accessControlPolicy.Owner.ID != parsedAcl.Owner { + debuglogger.Logf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.APIError{ + Code: "InvalidArgument", + Description: "Invalid id", + HTTPStatusCode: http.StatusBadRequest, + } + } + + if grants+acl != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", + grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrUnexpectedContent) + } + + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &accessControlPolicy, + } + } else if acl != "" { + if acl != "private" && acl != "public-read" && acl != "public-read-write" { + debuglogger.Logf("invalid acl: %q", acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + if grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", + grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) + } + + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACL(acl), + } + } else if grants != "" { + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + } + } else { + debuglogger.Logf("none of the bucket acl options has been specified: canned, req headers, req body") + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMissingSecurityHeader) + } + + updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam, acct.Role == auth.RoleAdmin) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + lockEnabled := strings.EqualFold(ctx.Get("X-Amz-Bucket-Object-Lock-Enabled"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + objectOwnership := types.ObjectOwnership( + ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), + ) + + // validate the bucket name + if ok := utils.IsValidBucketName(bucket); !ok { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + }, + }, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + + // validate the object ownership value + if ok := utils.IsValidOwnership(objectOwnership); !ok { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + }, + }, s3err.APIError{ + Code: "InvalidArgument", + Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership), + HTTPStatusCode: http.StatusBadRequest, + } + } + + if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled for %v object ownership", objectOwnership) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership) + } + + if acl != "" && grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) + } + + defACL := auth.ACL{ + Owner: acct.Access, + } + + updAcl, err := auth.UpdateACL(&auth.PutBucketAclInput{ + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + AccessControlPolicy: &auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: &acct.Access, + }}, + ACL: types.BucketCannedACL(acl), + }, defACL, c.iam, acct.Role == auth.RoleAdmin) + if err != nil { + debuglogger.Logf("failed to update bucket acl: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, err + } + + err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{ + Bucket: &bucket, + ObjectOwnership: objectOwnership, + ObjectLockEnabledForBucket: &lockEnabled, + }, updAcl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, err +} diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go new file mode 100644 index 00000000..a26b032b --- /dev/null +++ b/s3api/controllers/object-head.go @@ -0,0 +1,160 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + // url values + bucket := ctx.Params("bucket") + partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) + versionId := ctx.Query("versionId") + objRange := ctx.Get("Range") + key := ctx.Params("key") + keyEnd := ctx.Params("*1") + if keyEnd != "" { + key = strings.Join([]string{key, keyEnd}, "/") + } + path := ctx.Path() + if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { + key = key + "/" + } + + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < 1 || partNumberQuery > 10000 { + debuglogger.Logf("invalid part number: %d", partNumberQuery) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + partNumber = &partNumberQuery + } + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") + } + + res, err := c.be.HeadObject(ctx.Context(), + &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, + VersionId: &versionId, + ChecksumMode: checksumMode, + Range: &objRange, + }) + if err != nil { + var headers map[string]*string + if res != nil { + headers = map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(res.LastModified.UTC().Format(timefmt)), + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Set the metadata headers + utils.SetMetaHeaders(ctx, res.Metadata) + + return &Response{ + Headers: map[string]*string{ + "ETag": res.ETag, + "x-amz-restore": res.Restore, + "accept-ranges": res.AcceptRanges, + "Content-Range": res.ContentRange, + "Content-Disposition": res.ContentDisposition, + "Content-Encoding": res.ContentEncoding, + "Content-Language": res.ContentLanguage, + "Cache-Control": res.CacheControl, + "Expires": res.ExpiresString, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "Content-Type": res.ContentType, + "x-amz-version-id": res.VersionId, + "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), + "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), + "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), + "x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus), + "x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass), + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, nil +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go index 97d66a70..f7b579ec 100644 --- a/s3api/controllers/utilities.go +++ b/s3api/controllers/utilities.go @@ -50,7 +50,7 @@ type MetaOptions struct { type Response struct { Data any - Headers map[string]string + Headers map[string]*string MetaOpts *MetaOptions } @@ -65,6 +65,10 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } response, err := handler(ctx) + + // Set the response headers + SetResponseHeaders(ctx, response.Headers) + opts := response.MetaOpts // Send the metrics if mm != nil { @@ -98,9 +102,6 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) } - // Set the response headers - SetResponseHeaders(ctx, response.Headers) - if opts.Status == 0 { opts.Status = http.StatusOK } @@ -171,11 +172,14 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } } -func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]string) { +func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { if headers == nil { return } for key, val := range headers { - ctx.Response().Header.Add(key, val) + if val == nil || *val == "" { + continue + } + ctx.Response().Header.Add(key, *val) } } diff --git a/s3api/router.go b/s3api/router.go index e6150dd6..70aaabdd 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -57,7 +57,14 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) // Put bucket operations - app.Put("/:bucket", ctrl.PutBucketActions) + app.Put("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.PutObjectLockConfiguration, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.PutBucketCors, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.PutBucketPolicy, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutBucketAcl, logger, evs, mm)) + app.Put("/:bucket", controllers.ProcessResponse(ctrl.CreateBucket, logger, evs, mm)) // HeadBucket app.Head("/:bucket", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) @@ -82,8 +89,8 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) - // HeadObject action - app.Head("/:bucket/:key/*", ctrl.HeadObject) + // HeadObject + app.Head("/:bucket/:key/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) // GetObjectAcl action // GetObject action diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index ada265c2..bf70fbf4 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -664,3 +664,37 @@ func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) { return tagSet, nil } + +// Returns the provided string pointer +func GetStringPtr(str string) *string { + if str == "" { + return nil + } + + return &str +} + +// Converts any type to a string pointer +func ConvertToStringPtr[T any](val T) *string { + str := fmt.Sprint(val) + return &str +} + +// Converst any pointer to a string pointer +func ConvertPtrToStringPtr[T any](val *T) *string { + if val == nil { + return nil + } + str := fmt.Sprint(*val) + return &str +} + +// Formats the date with the given formatting and returns a string pointer +func FormatDatePtrToString(date *time.Time, format string) *string { + if date == nil { + return nil + } + + formatted := date.UTC().Format(format) + return &formatted +} From 56d4e4aa3ef306fd18df6bce041598ee17a4bbfb Mon Sep 17 00:00:00 2001 From: niksis02 Date: Tue, 17 Jun 2025 21:01:16 +0400 Subject: [PATCH 07/20] feat: implements advanced routing for object GET actions. --- s3api/controllers/base.go | 600 ------------------------------- s3api/controllers/base_test.go | 203 ----------- s3api/controllers/object-get.go | 534 +++++++++++++++++++++++++++ s3api/controllers/object-head.go | 11 +- s3api/router.go | 22 +- 5 files changed, 548 insertions(+), 822 deletions(-) create mode 100644 s3api/controllers/object-get.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 3f8ef249..fae7a415 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - "math" "net/http" "net/url" "os" @@ -73,605 +72,6 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } -func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - partNumberMarker := ctx.Query("part-number-marker") - acceptRange := ctx.Get("Range") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - versionId := ctx.Query("versionId") - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - if ctx.Request().URI().QueryArgs().Has("tagging") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectTaggingAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - tags, err := c.be.GetObjectTagging(ctx.Context(), bucket, key) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - res := s3response.Tagging{ - TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, - } - - for key, val := range tags { - res.TagSet.Tags = append(res.TagSet.Tags, - s3response.Tag{Key: key, Value: val}) - } - - return SendXMLResponse(ctx, res, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("retention") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectRetentionAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetObjectRetention(ctx.Context(), bucket, key, versionId) - if err != nil { - return SendXMLResponse(ctx, data, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - retention, err := auth.ParseObjectLockRetentionOutput(data) - return SendXMLResponse(ctx, retention, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("legal-hold") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectLegalHoldAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - data, err := c.be.GetObjectLegalHold(ctx.Context(), bucket, key, versionId) - return SendXMLResponse(ctx, auth.ParseObjectLegalHoldOutput(data), err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if uploadId != "" { - if partNumberMarker != "" { - n, err := strconv.Atoi(partNumberMarker) - if err != nil || n < 0 { - if err != nil && c.debug { - debuglogger.Logf("error parsing part number marker %q: %v", - partNumberMarker, err) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListParts, - BucketOwner: parsedAcl.Owner, - }) - } - } - mxParts := ctx.Query("max-parts") - maxParts, err := utils.ParseUint(mxParts) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max parts %q: %v", - mxParts, err) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidMaxParts), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListParts, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.ListMultipartUploadPartsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListParts, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{ - Bucket: &bucket, - Key: &key, - UploadId: &uploadId, - PartNumberMarker: &partNumberMarker, - MaxParts: &maxParts, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionListParts, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("acl") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionReadAcp, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectAclAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{ - Bucket: &bucket, - Key: &key, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("attributes") { - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectAttributesAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAttributes, - BucketOwner: parsedAcl.Owner, - }) - } - maxParts := ctx.Get("X-Amz-Max-Parts") - partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker") - maxPartsParsed, err := utils.ParseUint(maxParts) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing max parts %q: %v", - maxParts, err) - } - return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAttributes, - BucketOwner: parsedAcl.Owner, - }) - } - attrs, err := utils.ParseObjectAttributes(ctx) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAttributes, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.GetObjectAttributes(ctx.Context(), - &s3.GetObjectAttributesInput{ - Bucket: &bucket, - Key: &key, - PartNumberMarker: &partNumberMarker, - MaxParts: &maxPartsParsed, - VersionId: &versionId, - }) - if err != nil { - hdrs := []utils.CustomHeader{} - - if res.DeleteMarker != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-delete-marker", - Value: "true", - }) - } - if getstring(res.VersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAttributes, - BucketOwner: parsedAcl.Owner, - }) - } - - hdrs := []utils.CustomHeader{} - - if getstring(res.VersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - if res.DeleteMarker != nil && *res.DeleteMarker { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-delete-marker", - Value: "true", - }) - } - if res.LastModified != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Last-Modified", - Value: res.LastModified.UTC().Format(iso8601TimeFormatExtended), - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - - return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObjectAttributes, - BucketOwner: parsedAcl.Owner, - }) - } - - action := auth.GetObjectAction - if versionId != "" { - action = auth.GetObjectVersionAction - } - - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: action, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObject, - BucketOwner: parsedAcl.Owner, - }) - } - - checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) - if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { - if c.debug { - debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) - } - return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObject, - BucketOwner: parsedAcl.Owner, - }) - } - - utils.ContextKeySkipResBodyLog.Set(ctx, true) - res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{ - Bucket: &bucket, - Key: &key, - Range: &acceptRange, - VersionId: &versionId, - ChecksumMode: checksumMode, - }) - if err != nil { - if res != nil { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-delete-marker", - Value: "true", - }, - { - Key: "Last-Modified", - Value: res.LastModified.UTC().Format(timefmt), - }, - }) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - contentType := getstring(res.ContentType) - if contentType == "" { - contentType = defaultContentType - } - acceptRanges := getstring(res.AcceptRanges) - if acceptRanges == "" { - acceptRanges = "bytes" - } - - hdrs := []utils.CustomHeader{ - { - Key: "Content-Type", - Value: contentType, - }, - { - Key: "ETag", - Value: getstring(res.ETag), - }, - { - Key: "accept-ranges", - Value: acceptRanges, - }, - } - if getstring(res.ContentDisposition) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Content-Disposition", - Value: getstring(res.ContentDisposition), - }) - } - if getstring(res.ContentEncoding) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }) - } - if getstring(res.ContentLanguage) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Content-Language", - Value: getstring(res.ContentLanguage), - }) - } - if getstring(res.CacheControl) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Cache-Control", - Value: getstring(res.CacheControl), - }) - } - if getstring(res.ExpiresString) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Expires", - Value: getstring(res.ExpiresString), - }) - } - if getstring(res.ContentRange) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Content-Range", - Value: getstring(res.ContentRange), - }) - } - if res.LastModified != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "Last-Modified", - Value: res.LastModified.UTC().Format(timefmt), - }) - } - if res.TagCount != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-tagging-count", - Value: fmt.Sprint(*res.TagCount), - }) - } - if res.StorageClass != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-storage-class", - Value: string(res.StorageClass), - }) - } - if res.ChecksumCRC32 != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - } - if res.ChecksumCRC32C != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - } - if res.ChecksumSHA1 != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - } - if res.ChecksumSHA256 != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - if res.ChecksumCRC64NVME != nil { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - } - if res.ChecksumType != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-type", - Value: string(res.ChecksumType), - }) - } - - // Set x-amz-meta-... headers - utils.SetMetaHeaders(ctx, res.Metadata) - // Set other response headers - utils.SetResponseHeaders(ctx, hdrs) - // Set version id header - if getstring(res.VersionId) != "" { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }, - }) - } - - status := http.StatusOK - if acceptRange != "" { - status = http.StatusPartialContent - } - - if res.Body != nil { - // -1 will stream response body until EOF if content length not set - contentLen := -1 - if res.ContentLength != nil { - if *res.ContentLength > int64(math.MaxInt) { - debuglogger.Logf("content length %v int overflow", - *res.ContentLength) - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRange), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObject, - BucketOwner: parsedAcl.Owner, - }) - } - contentLen = int(*res.ContentLength) - } - utils.StreamResponseBody(ctx, res.Body, contentLen) - } - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionGetObject, - ContentLength: getint64(res.ContentLength), - BucketOwner: parsedAcl.Owner, - Status: status, - }) -} - func getstring(s *string) string { if s == nil { return "" diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index d6192783..652e07e6 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -24,7 +24,6 @@ import ( "reflect" "strings" "testing" - "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -85,208 +84,6 @@ func TestNew(t *testing.T) { } } -func getPtr(val string) *string { - return &val -} - -func TestS3ApiController_GetActions(t *testing.T) { - type args struct { - req *http.Request - } - - now := time.Now() - - app := fiber.New() - contentLength := int64(1000) - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - ListPartsFunc: func(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) { - return s3response.ListPartsResult{}, nil - }, - GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { - return &s3.GetObjectAclOutput{}, nil - }, - GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { - return s3response.GetObjectAttributesResponse{}, nil - }, - GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - return &s3.GetObjectOutput{ - Metadata: map[string]string{"hello": "world"}, - ContentType: getPtr("application/xml"), - ContentEncoding: getPtr("gzip"), - ETag: getPtr("98sda7f97sa9df798sd79f8as9df"), - ContentLength: &contentLength, - LastModified: &now, - StorageClass: "storage class", - }, nil - }, - GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) { - return map[string]string{"hello": "world"}, nil - }, - GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { - result, err := json.Marshal(types.ObjectLockRetention{ - Mode: types.ObjectLockRetentionModeCompliance, - }) - if err != nil { - return nil, err - } - return result, nil - }, - GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { - result := true - return &result, nil - }, - }, - } - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Get("/:bucket/:key/*", s3ApiController.GetActions) - - // GetObjectAttributes success case - getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) - getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello") - - invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil) - invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Get-actions-get-tags-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key/key.json?tagging", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-get-object-retention-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-get-object-legal-hold-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-invalid-max-parts-string", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=invalid", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Get-actions-invalid-max-parts-negative", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=-8", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Get-actions-invalid-part-number-marker-string", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=invalid", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Get-actions-invalid-part-number-marker-negative", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=-8", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Get-actions-list-object-parts-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=23", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-get-object-acl-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?acl", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-get-object-attributes-success", - app: app, - args: args{ - req: getObjAttrs, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Get-actions-get-object-invalid-checksum-mode", - app: app, - args: args{ - req: invalidChecksumMode, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Get-actions-get-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.GetActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.GetActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - }) - } -} - func TestS3ApiController_PutBucketActions(t *testing.T) { type args struct { req *http.Request diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go new file mode 100644 index 00000000..e8f16b0d --- /dev/null +++ b/s3api/controllers/object-get.go @@ -0,0 +1,534 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) GetObjectTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectTaggingAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectTagging(ctx.Context(), bucket, key) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + tags := s3response.Tagging{ + TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, + } + + for key, val := range data { + tags.TagSet.Tags = append(tags.TagSet.Tags, + s3response.Tag{Key: key, Value: val}) + } + + return &Response{ + Data: tags, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectRetention(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectRetentionAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectRetention(ctx.Context(), bucket, key, versionId) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + retention, err := auth.ParseObjectLockRetentionOutput(data) + return &Response{ + Data: retention, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectLegalHoldAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data, err := c.be.GetObjectLegalHold(ctx.Context(), bucket, key, versionId) + return &Response{ + Data: auth.ParseObjectLegalHoldOutput(data), + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectAcl(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionReadAcp, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectAclAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{ + Bucket: &bucket, + Key: &key, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + uploadId := ctx.Query("uploadId") + partNumberMarker := ctx.Query("part-number-marker") + maxPartsStr := ctx.Query("max-parts") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + // parse the part number marker + if partNumberMarker != "" { + n, err := strconv.Atoi(partNumberMarker) + if err != nil || n < 0 { + debuglogger.Logf("invalid part number marker %q: %v", + partNumberMarker, err) + + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListParts, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker) + } + } + + // parse the max parts + maxParts, err := utils.ParseUint(maxPartsStr) + if err != nil { + debuglogger.Logf("error parsing max parts %q: %v", + maxPartsStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListParts, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxParts) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.ListMultipartUploadPartsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionListParts, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + PartNumberMarker: &partNumberMarker, + MaxParts: &maxParts, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionListParts, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + maxPartsStr := ctx.Get("X-Amz-Max-Parts") + partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectAttributesAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // parse max parts + maxParts, err := utils.ParseUint(maxPartsStr) + if err != nil { + debuglogger.Logf("error parsing max parts %q: %v", + maxPartsStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMaxParts) + } + + // parse the object attributes + attrs, err := utils.ParseObjectAttributes(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.GetObjectAttributes(ctx.Context(), + &s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &key, + PartNumberMarker: &partNumberMarker, + MaxParts: &maxParts, + VersionId: &versionId, + }) + if err != nil { + headers := map[string]*string{ + "x-amz-version-id": res.VersionId, + } + if res.DeleteMarker != nil && *res.DeleteMarker { + headers["x-amz-delete-marker"] = utils.GetStringPtr("true") + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + headers := map[string]*string{ + "x-amz-version-id": res.VersionId, + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, iso8601TimeFormatExtended), + } + if res.DeleteMarker != nil && *res.DeleteMarker { + headers["x-amz-delete-marker"] = utils.GetStringPtr("true") + } + + return &Response{ + Headers: headers, + Data: utils.FilterObjectAttributes(attrs, res), + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + acceptRange := ctx.Get("Range") + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + utils.ContextKeySkipResBodyLog.Set(ctx, true) + + action := auth.GetObjectAction + if versionId != "" { + action = auth.GetObjectVersionAction + } + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: action, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // validate the checksum mode + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") + } + + res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + Range: &acceptRange, + VersionId: &versionId, + ChecksumMode: checksumMode, + }) + if err != nil { + var headers map[string]*string + if res != nil { + headers = map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObjectAttributes, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Set x-amz-meta-... headers + utils.SetMetaHeaders(ctx, res.Metadata) + + status := http.StatusOK + if acceptRange != "" { + status = http.StatusPartialContent + } + + if res.Body != nil { + // -1 will stream response body until EOF if content length not set + contentLen := -1 + if res.ContentLength != nil { + if *res.ContentLength > int64(math.MaxInt) { + debuglogger.Logf("content length %v int overflow", + *res.ContentLength) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObject, + ContentLength: getint64(res.ContentLength), + BucketOwner: parsedAcl.Owner, + Status: status, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRange) + } + contentLen = int(*res.ContentLength) + } + utils.StreamResponseBody(ctx, res.Body, contentLen) + } + + return &Response{ + Headers: map[string]*string{ + "ETag": res.ETag, + "x-amz-restore": res.Restore, + "accept-ranges": res.AcceptRanges, + "Content-Range": res.ContentRange, + "Content-Disposition": res.ContentDisposition, + "Content-Encoding": res.ContentEncoding, + "Content-Language": res.ContentLanguage, + "Cache-Control": res.CacheControl, + "Expires": res.ExpiresString, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "Content-Type": res.ContentType, + "x-amz-version-id": res.VersionId, + "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), + "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), + "x-amz-tagging-count": utils.ConvertPtrToStringPtr(res.TagCount), + "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), + "x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus), + "x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass), + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionGetObject, + ContentLength: getint64(res.ContentLength), + BucketOwner: parsedAcl.Owner, + Status: status, + }, + }, nil +} diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index a26b032b..360eecca 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -15,6 +15,7 @@ package controllers import ( + "fmt" "strings" "time" @@ -39,15 +40,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) versionId := ctx.Query("versionId") objRange := ctx.Get("Range") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) var partNumber *int32 if ctx.Request().URI().QueryArgs().Has("partNumber") { diff --git a/s3api/router.go b/s3api/router.go index 70aaabdd..bc3187e9 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -54,9 +54,10 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets) } + // ListBuckets app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) - // Put bucket operations + // PUT bucket operations app.Put("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) app.Put("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) app.Put("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) @@ -76,7 +77,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Delete("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm)) app.Delete("/:bucket", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm)) - // Get bucket operations + // GET bucket operations app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm)) app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm)) app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm)) @@ -90,15 +91,16 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) // HeadObject - app.Head("/:bucket/:key/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) + app.Head("/:bucket/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) - // GetObjectAcl action - // GetObject action - // ListObjectParts action - // GetObjectTagging action - // ListParts action - // GetObjectAttributes action - app.Get("/:bucket/:key/*", ctrl.GetActions) + // GET object operations + app.Get("/:bucket/*", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetObjectTagging, logger, evs, mm)) + app.Get("/:bucket/*", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.GetObjectRetention, logger, evs, mm)) + app.Get("/:bucket/*", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.GetObjectLegalHold, logger, evs, mm)) + app.Get("/:bucket/*", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetObjectAcl, logger, evs, mm)) + app.Get("/:bucket/*", middlewares.MatchQueryArgs("attributes"), controllers.ProcessResponse(ctrl.GetObjectAttributes, logger, evs, mm)) + app.Get("/:bucket/*", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.ListParts, logger, evs, mm)) + app.Get("/:bucket/*", controllers.ProcessResponse(ctrl.GetObject, logger, evs, mm)) // DeleteObject action // AbortMultipartUpload action From a3fef4254ae43915952520cdf97d9f1eff61368d Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 18 Jun 2025 23:11:24 +0400 Subject: [PATCH 08/20] feat: implements advanced routing for object DELETE and POST actions. fixes #896 fixes #899 Registeres an all route matcher handler at the end of the router to handle the cases when the api call doesn't match to any s3 action. The all routes matcher returns `MethodNotAllowed` for this kind of requests. --- s3api/controllers/base.go | 379 ----------------------------- s3api/controllers/base_test.go | 264 -------------------- s3api/controllers/object-delete.go | 207 ++++++++++++++++ s3api/controllers/object-post.go | 376 ++++++++++++++++++++++++++++ s3api/controllers/utilities.go | 9 + s3api/router.go | 107 ++++---- 6 files changed, 648 insertions(+), 694 deletions(-) create mode 100644 s3api/controllers/object-delete.go create mode 100644 s3api/controllers/object-post.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index fae7a415..f3487983 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1938,385 +1938,6 @@ const ( timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" ) -func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - contentType := ctx.Get("Content-Type") - contentDisposition := ctx.Get("Content-Disposition") - contentLanguage := ctx.Get("Content-Language") - cacheControl := ctx.Get("Cache-Control") - contentEncoding := ctx.Get("Content-Encoding") - tagging := ctx.Get("X-Amz-Tagging") - - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - if ctx.Request().URI().QueryArgs().Has("restore") { - var restoreRequest types.RestoreRequest - if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil { - if !errors.Is(err, io.EOF) { - if c.debug { - debuglogger.Logf("failed to parse the request body: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionRestoreObject, - BucketOwner: parsedAcl.Owner, - }) - } - } - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.RestoreObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionRestoreObject, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.RestoreObject(ctx.Context(), &s3.RestoreObjectInput{ - Bucket: &bucket, - Key: &key, - RestoreRequest: &restoreRequest, - }) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionRestoreObject, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectRestoreCompleted, - }) - } - - if ctx.Request().URI().QueryArgs().Has("select") && ctx.Query("select-type") == "2" { - var payload s3response.SelectObjectContentPayload - - err := xml.Unmarshal(ctx.Body(), &payload) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling select object content: %v", err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionSelectObjectContent, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionSelectObjectContent, - BucketOwner: parsedAcl.Owner, - }) - } - - sw := c.be.SelectObjectContent(ctx.Context(), - &s3.SelectObjectContentInput{ - Bucket: &bucket, - Key: &key, - Expression: payload.Expression, - ExpressionType: payload.ExpressionType, - InputSerialization: payload.InputSerialization, - OutputSerialization: payload.OutputSerialization, - RequestProgress: payload.RequestProgress, - ScanRange: payload.ScanRange, - }) - - ctx.Context().SetBodyStreamWriter(sw) - - return nil - } - - if uploadId != "" { - data := struct { - Parts []types.CompletedPart `xml:"Part"` - }{} - - err := xml.Unmarshal(ctx.Body(), &data) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling complete multipart upload: %v", err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - if len(data.Parts) == 0 { - if c.debug { - debuglogger.Logf("empty parts provided for complete multipart upload") - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrEmptyParts), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - var mpuObjectSize *int64 - mpuObjSizeHdr := ctx.Get("X-Amz-Mp-Object-Size") - if mpuObjSizeHdr != "" { - val, err := strconv.ParseInt(mpuObjSizeHdr, 10, 64) - //TODO: Not sure if invalid request should be returned - if err != nil { - if c.debug { - debuglogger.Logf("invalid value for 'x-amz-mp-objects-size' header: %v", err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - if val < 0 { - debuglogger.Logf("value for 'x-amz-mp-objects-size' header is less than 0: %v", val) - return SendXMLResponse(ctx, nil, - s3err.GetInvalidMpObjectSizeErr(val), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - mpuObjectSize = &val - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - _, checksums, err := utils.ParseChecksumHeaders(ctx) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) - err = utils.IsChecksumTypeValid(checksumType) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(), - &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &key, - UploadId: &uploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: data.Parts, - }, - MpuObjectSize: mpuObjectSize, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), - ChecksumType: checksumType, - }) - if err == nil { - if versid != "" { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-version-id", - Value: versid, - }, - }) - } - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - ObjectETag: res.ETag, - EventName: s3event.EventCompleteMultipartUpload, - VersionId: backend.GetPtrFromString(versid), - }) - } - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCompleteMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.PutObjectAction, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - objLockState, err := utils.ParsObjectLockHdrs(ctx) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - metadata := utils.GetUserMetaData(&ctx.Request().Header) - - checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - expires := ctx.Get("Expires") - - res, err := c.be.CreateMultipartUpload(ctx.Context(), - s3response.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &key, - Tagging: &tagging, - ContentType: &contentType, - ContentEncoding: &contentEncoding, - ContentDisposition: &contentDisposition, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - ObjectLockRetainUntilDate: &objLockState.RetainUntilDate, - ObjectLockMode: objLockState.ObjectLockMode, - ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, - Metadata: metadata, - ChecksumAlgorithm: checksumAlgorithm, - ChecksumType: checksumType, - }) - if err == nil { - if checksumAlgorithm != "" { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-checksum-algorithm", - Value: string(checksumAlgorithm), - }, - }) - } - } - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) -} - type MetaOpts struct { Logger s3log.AuditLogger EvSender s3event.S3EventSender diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 652e07e6..65eb7e4e 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -15,7 +15,6 @@ package controllers import ( - "bufio" "context" "encoding/json" "fmt" @@ -837,269 +836,6 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { } } -func TestS3ApiController_DeleteActions(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { - return &s3.DeleteObjectOutput{}, nil - }, - AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error { - return nil - }, - DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error { - return nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions) - - // Error case - appErr := fiber.New() - - s3ApiControllerErr := S3ApiController{be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }} - - appErr.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - appErr.Delete("/:bucket/:key/*", s3ApiControllerErr.DeleteActions) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Abort-multipart-upload-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key?uploadId=324234", nil), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Remove-object-tagging-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key/key2?tagging", nil), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Delete-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Delete-object-error", - app: appErr, - args: args{ - req: httptest.NewRequest(http.MethodDelete, "/my-bucket/invalid-key", nil), - }, - wantErr: false, - statusCode: 404, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.DeleteActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.DeleteActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestS3ApiController_CreateActions(t *testing.T) { - type args struct { - req *http.Request - } - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - RestoreObjectFunc: func(context.Context, *s3.RestoreObjectInput) error { - return nil - }, - CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { - return s3response.CompleteMultipartUploadResult{}, "", nil - }, - CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { - return s3response.InitiateMultipartUploadResult{}, nil - }, - SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) { - return func(w *bufio.Writer) {} - }, - }, - } - - bdy := ` - - string - string - - ` - - completMpBody := ` - - - etag - 1 - - - ` - - completMpEmptyBody := ` - - ` - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Post("/:bucket/:key/*", s3ApiController.CreateActions) - - invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil) - invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Restore-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Select-object-content-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Select-object-content-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?select&select-type=2", strings.NewReader(bdy)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Complete-multipart-upload-error", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Complete-multipart-upload-empty-parts", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Complete-multipart-upload-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Create-multipart-upload-invalid-checksum-algorithm", - app: app, - args: args{ - req: invChecksumAlgo, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Create-multipart-upload-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.CreateActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.CreateActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - func Test_XMLresponse(t *testing.T) { type args struct { ctx *fiber.Ctx diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go new file mode 100644 index 00000000..2cbebae2 --- /dev/null +++ b/s3api/controllers/object-delete.go @@ -0,0 +1,207 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "fmt" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3event" +) + +func (c S3ApiController) DeleteObjectTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.DeleteObjectTaggingAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.DeleteObjectTagging(ctx.Context(), bucket, key) + return &Response{ + MetaOpts: &MetaOptions{ + Status: http.StatusNoContent, + Action: metrics.ActionDeleteObjectTagging, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectTaggingDelete, + }, + }, err +} + +func (c S3ApiController) AbortMultipartUplaod(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + uploadId := ctx.Query("uploadId") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.AbortMultipartUploadAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionAbortMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.AbortMultipartUpload(ctx.Context(), + &s3.AbortMultipartUploadInput{ + UploadId: &uploadId, + Bucket: &bucket, + Key: &key, + }) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionAbortMultipartUpload, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.DeleteObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess( + ctx.Context(), + bucket, + acct.Access, + []types.ObjectIdentifier{ + { + Key: &key, + VersionId: &versionId, + }, + }, + bypass, + IsBucketPublic, + c.be, + ) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.DeleteObject(ctx.Context(), + &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &key, + VersionId: &versionId, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDelete, + Status: http.StatusNoContent, + }, + }, err + } + + headers := map[string]*string{ + "x-amz-version-id": res.VersionId, + } + + if res.DeleteMarker != nil && *res.DeleteMarker { + headers["x-amz-delete-marker"] = utils.GetStringPtr("true") + } + + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObject, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDelete, + Status: http.StatusNoContent, + }, + }, nil +} diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go new file mode 100644 index 00000000..bbe96643 --- /dev/null +++ b/s3api/controllers/object-post.go @@ -0,0 +1,376 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + var restoreRequest types.RestoreRequest + if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil { + debuglogger.Logf("failed to parse the request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionRestoreObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.RestoreObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionRestoreObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.RestoreObject(ctx.Context(), &s3.RestoreObjectInput{ + Bucket: &bucket, + Key: &key, + RestoreRequest: &restoreRequest, + }) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionRestoreObject, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRestoreCompleted, + }, + }, err +} + +func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + var payload s3response.SelectObjectContentPayload + + err := xml.Unmarshal(ctx.Body(), &payload) + if err != nil { + debuglogger.Logf("error unmarshalling select object content: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionSelectObjectContent, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionSelectObjectContent, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + sw := c.be.SelectObjectContent(ctx.Context(), + &s3.SelectObjectContentInput{ + Bucket: &bucket, + Key: &key, + Expression: payload.Expression, + ExpressionType: payload.ExpressionType, + InputSerialization: payload.InputSerialization, + OutputSerialization: payload.OutputSerialization, + RequestProgress: payload.RequestProgress, + ScanRange: payload.ScanRange, + }) + + ctx.Context().SetBodyStreamWriter(sw) + + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionSelectObjectContent, + BucketOwner: parsedAcl.Owner, + }, + }, nil +} + +func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + contentType := ctx.Get("Content-Type") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") + contentEncoding := ctx.Get("Content-Encoding") + tagging := ctx.Get("X-Amz-Tagging") + expires := ctx.Get("Expires") + metadata := utils.GetUserMetaData(&ctx.Request().Header) + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + objLockState, err := utils.ParsObjectLockHdrs(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.CreateMultipartUpload(ctx.Context(), + s3response.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + Tagging: &tagging, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + ObjectLockRetainUntilDate: &objLockState.RetainUntilDate, + ObjectLockMode: objLockState.ObjectLockMode, + ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, + Metadata: metadata, + ChecksumAlgorithm: checksumAlgorithm, + ChecksumType: checksumType, + }) + var headers map[string]*string + if err == nil { + headers = map[string]*string{ + "x-amz-checksum-algorithm": utils.ConvertToStringPtr(checksumAlgorithm), + } + } + return &Response{ + Headers: headers, + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + uploadId := ctx.Query("uploadId") + mpuObjSizeHdr := ctx.Get("X-Amz-Mp-Object-Size") + checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + data := struct { + Parts []types.CompletedPart `xml:"Part"` + }{} + + err = xml.Unmarshal(ctx.Body(), &data) + if err != nil { + debuglogger.Logf("error unmarshalling complete multipart upload: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + if len(data.Parts) == 0 { + debuglogger.Logf("empty parts provided for complete multipart upload") + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrEmptyParts) + } + + var mpuObjectSize *int64 + if mpuObjSizeHdr != "" { + val, err := strconv.ParseInt(mpuObjSizeHdr, 10, 64) + //TODO: Not sure if invalid request should be returned + if err != nil { + debuglogger.Logf("invalid value for 'x-amz-mp-objects-size' header: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + if val < 0 { + debuglogger.Logf("value for 'x-amz-mp-objects-size' header is less than 0: %v", val) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidMpObjectSizeErr(val) + } + + mpuObjectSize = &val + } + + _, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = utils.IsChecksumTypeValid(checksumType) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(), + &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: data.Parts, + }, + MpuObjectSize: mpuObjectSize, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumType: checksumType, + }) + return &Response{ + Data: res, + Headers: map[string]*string{ + "x-amz-version-id": &versid, + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + ObjectETag: res.ETag, + EventName: s3event.EventCompleteMultipartUpload, + VersionId: &versid, + }, + }, err +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go index f7b579ec..097b747b 100644 --- a/s3api/controllers/utilities.go +++ b/s3api/controllers/utilities.go @@ -183,3 +183,12 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { ctx.Response().Header.Add(key, *val) } } + +// Returns MethodNotAllowed for unmatched routes +func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUndetected, + }, + }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) +} diff --git a/s3api/router.go b/s3api/router.go index bc3187e9..8014e951 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -54,67 +54,69 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets) } - // ListBuckets + // ListBuckets action app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) + bucketRouter := app.Group("/:bucket") + objectRouter := app.Group("/:bucket/*") + // PUT bucket operations - app.Put("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.PutObjectLockConfiguration, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.PutBucketCors, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.PutBucketPolicy, logger, evs, mm)) - app.Put("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutBucketAcl, logger, evs, mm)) - app.Put("/:bucket", controllers.ProcessResponse(ctrl.CreateBucket, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.PutObjectLockConfiguration, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.PutBucketCors, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.PutBucketPolicy, logger, evs, mm)) + bucketRouter.Put("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutBucketAcl, logger, evs, mm)) + bucketRouter.Put("", controllers.ProcessResponse(ctrl.CreateBucket, logger, evs, mm)) - // HeadBucket - app.Head("/:bucket", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) + // HeadBucket action + bucketRouter.Head("", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) - // Delete bucket operations - app.Delete("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteBucketTagging, logger, evs, mm)) - app.Delete("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.DeleteBucketOwnershipControls, logger, evs, mm)) - app.Delete("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.DeleteBucketPolicy, logger, evs, mm)) - app.Delete("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm)) - app.Delete("/:bucket", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm)) + // DELETE bucket operations + bucketRouter.Delete("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteBucketTagging, logger, evs, mm)) + bucketRouter.Delete("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.DeleteBucketOwnershipControls, logger, evs, mm)) + bucketRouter.Delete("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.DeleteBucketPolicy, logger, evs, mm)) + bucketRouter.Delete("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm)) + bucketRouter.Delete("", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm)) // GET bucket operations - app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.GetBucketPolicy, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.GetBucketCors, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.GetObjectLockConfiguration, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetBucketAcl, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.ListMultipartUploads, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(ctrl.ListObjectVersions, logger, evs, mm)) - app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) - app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) - - // HeadObject - app.Head("/:bucket/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) - - // GET object operations - app.Get("/:bucket/*", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetObjectTagging, logger, evs, mm)) - app.Get("/:bucket/*", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.GetObjectRetention, logger, evs, mm)) - app.Get("/:bucket/*", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.GetObjectLegalHold, logger, evs, mm)) - app.Get("/:bucket/*", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetObjectAcl, logger, evs, mm)) - app.Get("/:bucket/*", middlewares.MatchQueryArgs("attributes"), controllers.ProcessResponse(ctrl.GetObjectAttributes, logger, evs, mm)) - app.Get("/:bucket/*", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.ListParts, logger, evs, mm)) - app.Get("/:bucket/*", controllers.ProcessResponse(ctrl.GetObject, logger, evs, mm)) - - // DeleteObject action - // AbortMultipartUpload action - // DeleteObjectTagging action - app.Delete("/:bucket/:key/*", ctrl.DeleteActions) + bucketRouter.Get("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.GetBucketPolicy, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.GetBucketCors, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.GetObjectLockConfiguration, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetBucketAcl, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.ListMultipartUploads, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(ctrl.ListObjectVersions, logger, evs, mm)) + bucketRouter.Get("", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) + bucketRouter.Get("", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) // DeleteObjects action - app.Post("/:bucket", ctrl.DeleteObjects) + bucketRouter.Post("", ctrl.DeleteObjects) - // CompleteMultipartUpload action - // CreateMultipartUpload - // RestoreObject action - // SelectObjectContent action - app.Post("/:bucket/:key/*", ctrl.CreateActions) + // HeadObject + objectRouter.Head("", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) + + // GET object operations + objectRouter.Get("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetObjectTagging, logger, evs, mm)) + objectRouter.Get("", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.GetObjectRetention, logger, evs, mm)) + objectRouter.Get("", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.GetObjectLegalHold, logger, evs, mm)) + objectRouter.Get("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetObjectAcl, logger, evs, mm)) + objectRouter.Get("", middlewares.MatchQueryArgs("attributes"), controllers.ProcessResponse(ctrl.GetObjectAttributes, logger, evs, mm)) + objectRouter.Get("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.ListParts, logger, evs, mm)) + objectRouter.Get("", controllers.ProcessResponse(ctrl.GetObject, logger, evs, mm)) + + // DELETE object operations + objectRouter.Delete("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteObjectTagging, logger, evs, mm)) + objectRouter.Delete("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.AbortMultipartUplaod, logger, evs, mm)) + objectRouter.Delete("", controllers.ProcessResponse(ctrl.DeleteObject, logger, evs, mm)) + + objectRouter.Post("", middlewares.MatchQueryArgs("restore"), controllers.ProcessResponse(ctrl.RestoreObject, logger, evs, mm)) + objectRouter.Post("", middlewares.MatchQueryArgs("list-type"), middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.RestoreObject, logger, evs, mm)) + objectRouter.Post("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.CompleteMultipartUpload, logger, evs, mm)) + objectRouter.Post("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.CreateMultipartUpload, logger, evs, mm)) // CopyObject action // PutObject action @@ -123,4 +125,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ // PutObjectTagging action // PutObjectAcl action app.Put("/:bucket/:key/*", ctrl.PutActions) + + // Return MethodNotAllowed for all the unmatched routes + app.All("*", controllers.ProcessResponse(ctrl.HandleUnmatch, logger, evs, mm)) } From b7c758b06577df5270c90dbb107ab5c8dcf915a4 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 19 Jun 2025 22:08:54 +0400 Subject: [PATCH 09/20] feat: implements advanced routing for bucket POST and object PUT operations. Fixes #1036 Fixes the issue when calling a non-existing root endpoint(POST /) the gateway returns `NoSuchBucket`. Now it returns the correct `MethodNotAllowed` error. --- backend/posix/posix.go | 8 + s3api/controllers/base.go | 8 - s3api/controllers/base_test.go | 1001 ------------------------- s3api/controllers/bucket-post.go | 100 +++ s3api/controllers/object-put.go | 860 +++++++++++++++++++++ s3api/controllers/utilities.go | 1 + s3api/middlewares/acl-parser.go | 2 +- s3api/middlewares/router-utilities.go | 18 + s3api/router.go | 18 +- tests/integration/group-tests.go | 1 + 10 files changed, 999 insertions(+), 1018 deletions(-) delete mode 100644 s3api/controllers/base_test.go create mode 100644 s3api/controllers/bucket-post.go create mode 100644 s3api/controllers/object-put.go diff --git a/backend/posix/posix.go b/backend/posix/posix.go index d8660a23..c71874b3 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -2419,6 +2419,10 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. if errors.Is(err, syscall.EDQUOT) { return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } + // Return the error itself, if it's an 's3err.APIError' + if _, ok := err.(s3err.APIError); ok { + return nil, err + } return nil, fmt.Errorf("write part data: %w", err) } @@ -2853,6 +2857,10 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3 if errors.Is(err, syscall.EDQUOT) { return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } + // Return the error itself, if it's an 's3err.APIError' + if _, ok := err.(s3err.APIError); ok { + return s3response.PutObjectOutput{}, err + } return s3response.PutObjectOutput{}, fmt.Errorf("write object data: %w", err) } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index f3487983..34c36b46 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -72,13 +72,6 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } -func getstring(s *string) string { - if s == nil { - return "" - } - return *s -} - func getint64(i *int64) int64 { if i == nil { return 0 @@ -1974,7 +1967,6 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { ctx.Status(apierr.HTTPStatusCode) return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", "")) } - fmt.Println(err, "------------") fmt.Fprintf(os.Stderr, "Internal Error, %v\n", err) ctx.Status(http.StatusInternalServerError) diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go deleted file mode 100644 index 65eb7e4e..00000000 --- a/s3api/controllers/base_test.go +++ /dev/null @@ -1,1001 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package controllers - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "reflect" - "strings" - "testing" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/gofiber/fiber/v2" - "github.com/valyala/fasthttp" - "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/s3api/utils" - "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3response" -) - -var ( - acl auth.ACL - acldata []byte -) - -func init() { - var err error - acldata, err = json.Marshal(acl) - if err != nil { - panic(err) - } -} - -func TestNew(t *testing.T) { - type args struct { - be backend.Backend - iam auth.IAMService - } - - be := backend.BackendUnsupported{} - - tests := []struct { - name string - args args - want S3ApiController - }{ - { - name: "Initialize S3 api controller", - args: args{ - be: be, - iam: &auth.IAMServiceInternal{}, - }, - want: S3ApiController{ - be: be, - iam: &auth.IAMServiceInternal{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("New() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestS3ApiController_PutBucketActions(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - - // Mock valid acl - acl := auth.ACL{Owner: "valid access"} - acldata, err := json.Marshal(acl) - if err != nil { - t.Errorf("Failed to parse the params: %v", err.Error()) - return - } - - body := ` - - - - - hell - - string - - - - hello - - - ` - - invOwnerBody := ` - - - hello - - - ` - - tagBody := ` - - - - organization - marketing - - - - ` - - versioningBody := ` - - Enabled - Enabled - - ` - - policyBody := `{ - "Statement": [ - { - "Effect": "Allow", - "Principal": "*", - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::my-bucket/*" - } - ] - } - ` - - objectLockBody := ` - - Enabled - - - GOVERNANCE - 2 - - - - ` - - ownershipBody := ` - - - BucketOwnerEnforced - - - ` - - invalidOwnershipBody := ` - - - invalid_value - - - ` - - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - PutBucketAclFunc: func(context.Context, string, []byte) error { - return nil - }, - CreateBucketFunc: func(context.Context, *s3.CreateBucketInput, []byte) error { - return nil - }, - PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { - return nil - }, - PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { - return nil - }, - PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error { - return nil - }, - PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error { - return nil - }, - PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error { - return nil - }, - GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { - return types.ObjectOwnershipBucketOwnerPreferred, nil - }, - }, - } - // Mock ctx.Locals - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{Owner: "valid access"}) - return ctx.Next() - }) - app.Put("/:bucket", s3ApiController.PutBucketActions) - - // invalid acl case - invAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - invAclReq.Header.Set("X-Amz-Acl", "invalid") - - // invalid acl case 2 - errAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - errAclReq.Header.Set("X-Amz-Acl", "private") - errAclReq.Header.Set("X-Amz-Grant-Read", "hello") - - // PutBucketAcl incorrect bucket owner case - incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody)) - - // PutBucketAcl acl success - aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - aclSuccReq.Header.Set("X-Amz-Acl", "private") - - // Invalid acl body case - errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body)) - errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello") - - invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil) - invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Put-bucket-tagging-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-tagging-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Put-bucket-ownership-controls-invalid-ownership", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-ownership-controls-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-lock-configuration-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-lock-configuration-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-versioning-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-versioning-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", strings.NewReader(versioningBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-policy-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-policy-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", strings.NewReader(policyBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-acl-invalid-acl", - app: app, - args: args{ - req: invAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-acl", - app: app, - args: args{ - req: errAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-acl-body", - app: app, - args: args{ - req: errAclBodyReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-bucket-owner", - app: app, - args: args{ - req: incorrectBucketOwner, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-success", - app: app, - args: args{ - req: aclSuccReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Create-bucket-invalid-acl-ownership-combination", - app: app, - args: args{ - req: invAclOwnershipReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Create-bucket-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.PutBucketActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.PutBucketActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestS3ApiController_PutActions(t *testing.T) { - type args struct { - req *http.Request - } - - body := ` - - - - - hell - - string - - - - hello - - - ` - tagBody := ` - - - - string - string - - - - ` - - //retentionBody := ` - // - // GOVERNANCE - // 2025-01-01T00:00:00Z - // - //` - - legalHoldBody := ` - - ON - - ` - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error { - return nil - }, - CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) { - return s3response.CopyObjectOutput{ - CopyObjectResult: &s3response.CopyObjectResult{}, - }, nil - }, - PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) { - return s3response.PutObjectOutput{}, nil - }, - UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { - return &s3.UploadPartOutput{}, nil - }, - PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error { - return nil - }, - UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { - return s3response.CopyPartResult{}, nil - }, - PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error { - return nil - }, - PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error { - return nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }, - } - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Put("/:bucket/:key/*", s3ApiController.PutActions) - - // UploadPartCopy success - uploadPartCpyReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=12asd32&partNumber=3", nil) - uploadPartCpyReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // UploadPartCopy error case - uploadPartCpyErrReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=12asd32&partNumber=invalid", nil) - uploadPartCpyErrReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // CopyObject success - cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // CopyObject invalid checksum algorithm - cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") - - // PutObjectAcl success - aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - aclReq.Header.Set("X-Amz-Acl", "private") - - // PutObjectAcl success grt case - aclGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - aclGrtReq.Header.Set("X-Amz-Grant-Read", "private") - - // invalid acl case 1 - invAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", nil) - invAclReq.Header.Set("X-Amz-Acl", "invalid") - - // invalid acl case 2 - errAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", nil) - errAclReq.Header.Set("X-Amz-Acl", "private") - errAclReq.Header.Set("X-Amz-Grant-Read", "hello") - - // invalid body & grt case - invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)) - invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello") - - // PutObject invalid checksum algorithm - invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") - - // PutObject invalid base64 checksum - invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64") - - // PutObject invalid crc32 - invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm") - - // PutObject invalid crc32c - invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==") - - // PutObject invalid sha1 - invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==") - - // PutObject invalid sha256 - invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm") - - // PutObject multiple checksum headers - mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=") - mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") - - // PutObject checksum algorithm and header mismatch - checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1") - checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Put-object-part-error-case", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=abc&partNumber=invalid", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-part-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=4&partNumber=3", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Set-tags-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?tagging", strings.NewReader(tagBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "put-object-retention-invalid-request", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil), - }, - wantErr: false, - statusCode: 400, - }, - //{ - // name: "put-object-retention-success", - // app: app, - // args: args{ - // req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)), - // }, - // wantErr: false, - // statusCode: 200, - //}, - { - name: "put-legal-hold-invalid-request", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "put-legal-hold-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-invalid-acl", - app: app, - args: args{ - req: invAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-incorrect-acl", - app: app, - args: args{ - req: errAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-incorrect-acl-body-case", - app: app, - args: args{ - req: invAclBodyGrtReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-success", - app: app, - args: args{ - req: aclReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-success-body-case", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-success-grt-case", - app: app, - args: args{ - req: aclGrtReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Upload-part-copy-invalid-part-number", - app: app, - args: args{ - req: uploadPartCpyErrReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Upload-part-copy-success", - app: app, - args: args{ - req: uploadPartCpyReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Copy-object-invalid-checksum-algorithm", - app: app, - args: args{ - req: cpyInvChecksumAlgo, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Copy-object-success", - app: app, - args: args{ - req: cpySrcReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key/key2", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.PutActions() %v error = %v, wantErr %v", - tt.name, err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.PutActions() %v statusCode = %v, wantStatusCode = %v", - tt.name, resp.StatusCode, tt.statusCode) - } - } -} - -func TestS3ApiController_DeleteObjects(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { - return s3response.DeleteResult{}, nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Post("/:bucket", s3ApiController.DeleteObjects) - - // Valid request body - xmlBody := `body` - - request := httptest.NewRequest(http.MethodPost, "/my-bucket", strings.NewReader(xmlBody)) - request.Header.Set("Content-Type", "application/xml") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Delete-Objects-success", - app: app, - args: args{ - req: request, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Delete-Objects-error", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 400, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.DeleteObjects() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.DeleteObjects() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func Test_XMLresponse(t *testing.T) { - type args struct { - ctx *fiber.Ctx - resp any - err error - } - - app := fiber.New() - ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) - - tests := []struct { - name string - args args - wantErr bool - statusCode int - }{ - { - name: "Internal-server-error", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrInternalError), - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Error-not-implemented", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrNotImplemented), - }, - wantErr: false, - statusCode: 501, - }, - { - name: "Invalid-request-body", - args: args{ - ctx: ctx, - resp: make(chan int), - err: nil, - }, - wantErr: true, - statusCode: 200, - }, - { - name: "Successful-response", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr { - t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) - } - - statusCode := tt.args.ctx.Response().StatusCode() - - if statusCode != tt.statusCode { - t.Errorf("response() %v code = %v, wantErr %v", tt.name, statusCode, tt.wantErr) - } - - tt.args.ctx.Status(http.StatusOK) - }) - } -} - -func Test_response(t *testing.T) { - type args struct { - ctx *fiber.Ctx - resp any - err error - opts *MetaOpts - } - - app := fiber.New() - ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) - - tests := []struct { - name string - args args - wantErr bool - statusCode int - }{ - { - name: "Internal-server-error", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrInternalError), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Internal-server-error-not-api", - args: args{ - ctx: ctx, - resp: nil, - err: fmt.Errorf("custom error"), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Error-not-implemented", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrNotImplemented), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 501, - }, - { - name: "Successful-response", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Successful-response-status-204", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - opts: &MetaOpts{ - Status: 204, - }, - }, - wantErr: false, - statusCode: 204, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := SendResponse(tt.args.ctx, tt.args.err, tt.args.opts); (err != nil) != tt.wantErr { - t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) - } - - statusCode := tt.args.ctx.Response().StatusCode() - - if statusCode != tt.statusCode { - t.Errorf("response() %v code = %v, wantErr %v", tt.name, statusCode, tt.wantErr) - } - }) - } -} diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go new file mode 100644 index 00000000..f9ce29cd --- /dev/null +++ b/s3api/controllers/bucket-post.go @@ -0,0 +1,100 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + var dObj s3response.DeleteObjects + + err := xml.Unmarshal(ctx.Body(), &dObj) + if err != nil { + debuglogger.Logf("error unmarshalling delete objects: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.DeleteObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.DeleteObjects(ctx.Context(), + &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: dObj.Objects, + }, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + ObjectCount: int64(len(dObj.Objects)), + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDeleteObjects, + }, + }, err +} diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go new file mode 100644 index 00000000..3e1de9fa --- /dev/null +++ b/s3api/controllers/object-put.go @@ -0,0 +1,860 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "fmt" + "io" + "net/url" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutBucketTaggingAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectTagging(ctx.Context(), bucket, key, tagging) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectTaggingPut, + }, + }, err +} + +func (c S3ApiController) PutObjectRetention(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectRetentionAction, + IsBucketPublic: IsBucketPublic, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if bypass { + policy, err := c.be.GetBucketPolicy(ctx.Context(), bucket) + if err != nil { + bypass = false + } else { + if err := auth.VerifyBucketPolicy(policy, acct.Access, bucket, key, auth.BypassGovernanceRetentionAction); err != nil { + bypass = false + } + } + } + + retention, err := auth.ParseObjectLockRetentionInput(ctx.Body()) + if err != nil { + debuglogger.Logf("failed to parse object lock configuration input: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectRetention(ctx.Context(), bucket, key, versionId, bypass, retention) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + var legalHold types.ObjectLockLegalHold + if err := xml.Unmarshal(ctx.Body(), &legalHold); err != nil { + debuglogger.Logf("failed to parse request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + if legalHold.Status != types.ObjectLockLegalHoldStatusOff && legalHold.Status != types.ObjectLockLegalHoldStatusOn { + debuglogger.Logf("invalid legal hold status: %v", legalHold.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectLegalHoldAction, + IsBucketPublic: IsBucketPublic, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err := c.be.PutObjectLegalHold(ctx.Context(), bucket, key, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + partNumber := int32(ctx.QueryInt("partNumber", -1)) + uploadId := ctx.Query("uploadId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + contentLengthStr := ctx.Get("Content-Length") + if contentLengthStr == "" { + contentLengthStr = "0" + } + // Use decoded content length if available because the + // middleware will decode the chunked transfer encoding + decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") + if decodedLength != "" { + contentLengthStr = decodedLength + } + + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + debuglogger.Logf("err parsing checksum headers: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var body io.Reader + bodyi := utils.ContextKeyBodyReader.Get(ctx) + if bodyi != nil { + body = bodyi.(io.Reader) + } else { + body = ctx.Request().BodyStream() + } + + res, err := c.be.UploadPart(ctx.Context(), + &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + PartNumber: &partNumber, + ContentLength: &contentLength, + Body: body, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + }) + var headers map[string]*string + if err == nil { + headers = map[string]*string{ + "ETag": res.ETag, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + ContentLength: contentLength, + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + +} + +func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") + copySrcRange := ctx.Get("X-Amz-Copy-Source-Range") + partNumber := int32(ctx.QueryInt("partNumber", -1)) + uploadId := ctx.Query("uploadId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + cs := copySource + copySource, err := url.QueryUnescape(copySource) + if err != nil { + debuglogger.Logf("error unescaping copy source %q: %v", cs, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + resp, err := c.be.UploadPartCopy(ctx.Context(), + &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &key, + CopySource: ©Source, + PartNumber: &partNumber, + UploadId: &uploadId, + CopySourceRange: ©SrcRange, + }) + var headers map[string]*string + if err == nil && resp.CopySourceVersionId != "" { + headers = map[string]*string{ + "x-amz-copy-source-version-id": &resp.CopySourceVersionId, + } + } + return &Response{ + Headers: headers, + Data: resp, + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAclAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var input *s3.PutObjectAclInput + if len(ctx.Body()) > 0 { + if grants+acl != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + var accessControlPolicy auth.AccessControlPolicy + err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) + if err != nil { + debuglogger.Logf("error unmarshalling access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + //TODO: This part will be changed when object acls are implemented + + grants := []types.Grant{} + for _, grt := range accessControlPolicy.AccessControlList.Grants { + grants = append(grants, types.Grant{ + Grantee: &types.Grantee{ + ID: &grt.Grantee.ID, + Type: grt.Grantee.Type, + }, + Permission: types.Permission(grt.Permission), + }) + } + + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + ACL: "", + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: accessControlPolicy.Owner, + Grants: grants, + }, + } + } + if acl != "" { + if acl != "private" && acl != "public-read" && acl != "public-read-write" { + debuglogger.Logf("invalid acl: %q", acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + if len(ctx.Body()) > 0 || grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", grants, acl, len(ctx.Body())) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + ACL: types.ObjectCannedACL(acl), + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: &types.Owner{ID: &parsedAcl.Owner}, + }, + } + } + if grants != "" { + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: &types.Owner{ID: &parsedAcl.Owner}, + }, + ACL: "", + } + } + + err = c.be.PutObjectAcl(ctx.Context(), input) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectAclPut, + }, + }, err +} + +func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") + copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") + copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") + copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") + copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") + metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy))) + taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy))) + contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") + expires := ctx.Get("Expires") + tagging := ctx.Get("x-amz-tagging") + storageClass := ctx.Get("X-Amz-Storage-Class") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + cs := copySource + copySource, err := url.QueryUnescape(copySource) + if err != nil { + debuglogger.Logf("error unescaping copy source %q: %v", cs, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var mtime *time.Time + if copySrcModifSince != "" { + tm, err := time.Parse(iso8601Format, copySrcModifSince) + if err != nil { + debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + mtime = &tm + } + var umtime *time.Time + if copySrcUnmodifSince != "" { + tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) + if err != nil { + debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + umtime = &tm + } + + metadata := utils.GetUserMetaData(&ctx.Request().Header) + + if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace { + debuglogger.Logf("invalid metadata directive: %v", metaDirective) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMetadataDirective) + } + + if taggingDirective != "" && taggingDirective != types.TaggingDirectiveCopy && taggingDirective != types.TaggingDirectiveReplace { + debuglogger.Logf("invalid tagging direcrive: %v", taggingDirective) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidTaggingDirective) + } + + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + objLock, err := utils.ParsObjectLockHdrs(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.CopyObject(ctx.Context(), + s3response.CopyObjectInput{ + Bucket: &bucket, + Key: &key, + ContentType: &contentType, + ContentDisposition: &contentDisposition, + ContentEncoding: &contentEncoding, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + Tagging: &tagging, + TaggingDirective: taggingDirective, + CopySource: ©Source, + CopySourceIfMatch: ©SrcIfMatch, + CopySourceIfNoneMatch: ©SrcIfNoneMatch, + CopySourceIfModifiedSince: mtime, + CopySourceIfUnmodifiedSince: umtime, + ExpectedBucketOwner: &acct.Access, + Metadata: metadata, + MetadataDirective: metaDirective, + StorageClass: types.StorageClass(storageClass), + ChecksumAlgorithm: checksumAlgorithm, + ObjectLockRetainUntilDate: &objLock.RetainUntilDate, + ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ObjectLockMode: objLock.ObjectLockMode, + }) + + var etag *string + if err == nil { + etag = res.CopyObjectResult.ETag + } + + return &Response{ + Headers: map[string]*string{ + "x-amz-version-id": res.VersionId, + "x-amz-copy-source-version-id": res.CopySourceVersionId, + }, + Data: res.CopyObjectResult, + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + ObjectETag: etag, + VersionId: res.VersionId, + EventName: s3event.EventObjectCreatedCopy, + }, + }, err +} + +func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") + expires := ctx.Get("Expires") + tagging := ctx.Get("x-amz-tagging") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + // Content Length + contentLengthStr := ctx.Get("Content-Length") + if contentLengthStr == "" { + contentLengthStr = "0" + } + // Use decoded content length if available because the + // middleware will decode the chunked transfer encoding + decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") + if decodedLength != "" { + contentLengthStr = decodedLength + } + + // load the meta headers + metadata := utils.GetUserMetaData(&ctx.Request().Header) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, IsBucketPublic, c.be) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + objLock, err := utils.ParsObjectLockHdrs(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var body io.Reader + bodyi := utils.ContextKeyBodyReader.Get(ctx) + if bodyi != nil { + body = bodyi.(io.Reader) + } else { + body = ctx.Request().BodyStream() + } + + res, err := c.be.PutObject(ctx.Context(), + s3response.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ContentLength: &contentLength, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + Metadata: metadata, + Body: body, + Tagging: &tagging, + ObjectLockRetainUntilDate: &objLock.RetainUntilDate, + ObjectLockMode: objLock.ObjectLockMode, + ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + }) + return &Response{ + Headers: map[string]*string{ + "ETag": &res.ETag, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-version-id": &res.VersionID, + }, + MetaOpts: &MetaOptions{ + ContentLength: contentLength, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: contentLength, + EventName: s3event.EventObjectCreatedPut, + }, + }, err +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go index 097b747b..3e6d4345 100644 --- a/s3api/controllers/utilities.go +++ b/s3api/controllers/utilities.go @@ -172,6 +172,7 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } } +// Sets the response headers func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { if headers == nil { return diff --git a/s3api/middlewares/acl-parser.go b/s3api/middlewares/acl-parser.go index ba1fc21b..20a4a56f 100644 --- a/s3api/middlewares/acl-parser.go +++ b/s3api/middlewares/acl-parser.go @@ -38,7 +38,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe path := ctx.Path() pathParts := strings.Split(path, "/") bucket := pathParts[1] - if path == "/" && ctx.Method() == http.MethodGet { + if path == "/" { return ctx.Next() } if ctx.Method() == http.MethodPatch { diff --git a/s3api/middlewares/router-utilities.go b/s3api/middlewares/router-utilities.go index 2a09f251..47929103 100644 --- a/s3api/middlewares/router-utilities.go +++ b/s3api/middlewares/router-utilities.go @@ -19,6 +19,7 @@ import ( "github.com/versity/versitygw/s3api/utils" ) +// Evaluates/Matches the provided requst query params func MatchQueryArgs(args ...string) fiber.Handler { return func(ctx *fiber.Ctx) error { if utils.ContextKeySkip.IsSet(ctx) { @@ -34,6 +35,23 @@ func MatchQueryArgs(args ...string) fiber.Handler { } } +// Evaluates/Matches the requst header +func MatchHeader(key string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + if utils.ContextKeySkip.IsSet(ctx) { + return ctx.Next() + } + + val := ctx.Get(key) + if val == "" { + utils.ContextKeySkip.Set(ctx, true) + } + + return ctx.Next() + } +} + +// Evaluates/Matches the requst query param and value func MatchQueryArgWithValue(key, val string) fiber.Handler { return func(ctx *fiber.Ctx) error { if utils.ContextKeySkip.IsSet(ctx) { diff --git a/s3api/router.go b/s3api/router.go index 8014e951..31aa0ced 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -94,7 +94,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ bucketRouter.Get("", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) // DeleteObjects action - bucketRouter.Post("", ctrl.DeleteObjects) + bucketRouter.Post("", middlewares.MatchQueryArgs("delete"), controllers.ProcessResponse(ctrl.DeleteObjects, logger, evs, mm)) // HeadObject objectRouter.Head("", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) @@ -118,13 +118,15 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ objectRouter.Post("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.CompleteMultipartUpload, logger, evs, mm)) objectRouter.Post("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.CreateMultipartUpload, logger, evs, mm)) - // CopyObject action - // PutObject action - // UploadPart action - // UploadPartCopy action - // PutObjectTagging action - // PutObjectAcl action - app.Put("/:bucket/:key/*", ctrl.PutActions) + // PUT object operations + objectRouter.Put("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutObjectTagging, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.PutObjectRetention, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.PutObjectLegalHold, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutObjectAcl, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.UploadPartCopy, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), controllers.ProcessResponse(ctrl.UploadPart, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.CopyObject, logger, evs, mm)) + objectRouter.Put("", controllers.ProcessResponse(ctrl.PutObject, logger, evs, mm)) // Return MethodNotAllowed for all the unmatched routes app.All("*", controllers.ProcessResponse(ctrl.HandleUnmatch, logger, evs, mm)) diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 6435e736..a3a15f50 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -917,6 +917,7 @@ func GetIntTests() IntTests { "PutObject_with_object_lock": PutObject_with_object_lock, "PutObject_invalid_legal_hold": PutObject_invalid_legal_hold, "PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode, + "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, "PutObject_invalid_checksum_header": PutObject_invalid_checksum_header, From abdf342ef7307383205d5b46422f282fde37bf2b Mon Sep 17 00:00:00 2001 From: niksis02 Date: Fri, 20 Jun 2025 23:23:53 +0400 Subject: [PATCH 10/20] feat: implements advanced routing for the admin apis. Adds the debug logging and quite mode for the separate admin server. Adjusts the admin apis to the new advanced routing changes. Enables debug logging for the separate admin server(when a separate server is run for the admin apis). Adds the quiet mode for the separate admin server. --- cmd/versitygw/main.go | 65 +++-- s3api/admin-router.go | 14 +- s3api/admin-server.go | 19 +- s3api/controllers/admin.go | 164 +++++++----- s3api/controllers/admin_test.go | 454 -------------------------------- s3api/controllers/base.go | 90 ------- s3api/middlewares/acl-parser.go | 4 +- s3api/router.go | 12 +- 8 files changed, 162 insertions(+), 660 deletions(-) delete mode 100644 s3api/controllers/admin_test.go diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 8ff4e620..f5b84efe 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -621,30 +621,6 @@ func runGateway(ctx context.Context, be backend.Backend) error { opts = append(opts, s3api.WithHostStyle(virtualDomain)) } - admApp := fiber.New(fiber.Config{ - AppName: "versitygw", - ServerHeader: "VERSITYGW", - Network: fiber.NetworkTCP, - DisableStartupMessage: true, - }) - - var admOpts []s3api.AdminOpt - - if admCertFile != "" || admKeyFile != "" { - if admCertFile == "" { - return fmt.Errorf("TLS key specified without cert file") - } - if admKeyFile == "" { - return fmt.Errorf("TLS cert specified without key file") - } - - cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile) - if err != nil { - return fmt.Errorf("tls: load certs: %v", err) - } - admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert)) - } - iam, err := auth.New(&auth.Opts{ RootAccount: auth.Account{ Access: rootUserAccess, @@ -732,7 +708,41 @@ func runGateway(ctx context.Context, be backend.Backend) error { return fmt.Errorf("init gateway: %v", err) } - admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...) + var admSrv *s3api.S3AdminServer + + if admPort != "" { + admApp := fiber.New(fiber.Config{ + AppName: "versitygw", + ServerHeader: "VERSITYGW", + Network: fiber.NetworkTCP, + DisableStartupMessage: true, + }) + + var opts []s3api.AdminOpt + + if admCertFile != "" || admKeyFile != "" { + if admCertFile == "" { + return fmt.Errorf("TLS key specified without cert file") + } + if admKeyFile == "" { + return fmt.Errorf("TLS cert specified without key file") + } + + cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile) + if err != nil { + return fmt.Errorf("tls: load certs: %v", err) + } + opts = append(opts, s3api.WithAdminSrvTLS(cert)) + } + if quiet { + opts = append(opts, s3api.WithAdminQuiet()) + } + if debug { + opts = append(opts, s3api.WithAdminDebug()) + } + + admSrv = s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, opts...) + } if !quiet { printBanner(port, admPort, certFile != "", admCertFile != "") @@ -977,10 +987,7 @@ func getMatchingIPs(spec string) ([]string, error) { const columnWidth = 70 func centerText(text string) string { - padding := (columnWidth - 2 - len(text)) / 2 - if padding < 0 { - padding = 0 - } + padding := max((columnWidth-2-len(text))/2, 0) return strings.Repeat(" ", padding) + text } diff --git a/s3api/admin-router.go b/s3api/admin-router.go index 65ec084a..16c2df52 100644 --- a/s3api/admin-router.go +++ b/s3api/admin-router.go @@ -25,23 +25,23 @@ import ( type S3AdminRouter struct{} func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) { - controller := controllers.NewAdminController(iam, be, logger) + ctrl := controllers.NewAdminController(iam, be, logger) // CreateUser admin api - app.Patch("/create-user", controller.CreateUser) + app.Patch("/create-user", controllers.ProcessResponse(ctrl.CreateUser, logger, nil, nil)) // DeleteUsers admin api - app.Patch("/delete-user", controller.DeleteUser) + app.Patch("/delete-user", controllers.ProcessResponse(ctrl.DeleteUser, logger, nil, nil)) // UpdateUser admin api - app.Patch("/update-user", controller.UpdateUser) + app.Patch("/update-user", controllers.ProcessResponse(ctrl.UpdateUser, logger, nil, nil)) // ListUsers admin api - app.Patch("/list-users", controller.ListUsers) + app.Patch("/list-users", controllers.ProcessResponse(ctrl.ListUsers, logger, nil, nil)) // ChangeBucketOwner admin api - app.Patch("/change-bucket-owner", controller.ChangeBucketOwner) + app.Patch("/change-bucket-owner", controllers.ProcessResponse(ctrl.ChangeBucketOwner, logger, nil, nil)) // ListBucketsAndOwners admin api - app.Patch("/list-buckets", controller.ListBuckets) + app.Patch("/list-buckets", controllers.ProcessResponse(ctrl.ListBuckets, logger, nil, nil)) } diff --git a/s3api/admin-server.go b/s3api/admin-server.go index ee132d74..23b66019 100644 --- a/s3api/admin-server.go +++ b/s3api/admin-server.go @@ -31,6 +31,8 @@ type S3AdminServer struct { router *S3AdminRouter port string cert *tls.Certificate + quiet bool + debug bool } func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer { @@ -46,8 +48,13 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse } // Logging middlewares - app.Use(logger.New()) + if !server.quiet { + app.Use(logger.New(logger.Config{ + Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n", + })) + } app.Use(middlewares.DecodeURL(l, nil)) + app.Use(middlewares.DebugLogger()) // Authentication middlewares app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false)) @@ -67,6 +74,16 @@ func WithAdminSrvTLS(cert tls.Certificate) AdminOpt { return func(s *S3AdminServer) { s.cert = &cert } } +// WithQuiet silences default logging output +func WithAdminQuiet() AdminOpt { + return func(s *S3AdminServer) { s.quiet = true } +} + +// WithAdminDebug enables the debug logging +func WithAdminDebug() AdminOpt { + return func(s *S3AdminServer) { s.debug = true } +} + func (sa *S3AdminServer) Serve() (err error) { if sa.cert != nil { return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert) diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index d10dae18..30cb6710 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -15,10 +15,13 @@ package controllers import ( + "encoding/json" "encoding/xml" + "fmt" "net/http" "strings" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" @@ -38,23 +41,23 @@ func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLo return AdminController{iam: iam, be: be, l: l} } -func (c AdminController) CreateUser(ctx *fiber.Ctx) error { +func (c AdminController) CreateUser(ctx *fiber.Ctx) (*Response, error) { var usr auth.Account err := xml.Unmarshal(ctx.Body(), &usr) if err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminCreateUser, - }) + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) } if !usr.Role.IsValid() { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminCreateUser, - }) + }, + }, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole) } err = c.iam.CreateAccount(usr) @@ -63,47 +66,47 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error { err = s3err.GetAPIError(s3err.ErrAdminUserExists) } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminCreateUser, - }) + }, + }, err } - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminCreateUser, Status: http.StatusCreated, - }) + }, + }, nil } -func (c AdminController) UpdateUser(ctx *fiber.Ctx) error { +func (c AdminController) UpdateUser(ctx *fiber.Ctx) (*Response, error) { access := ctx.Query("access") if access == "" { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminUpdateUser, - }) + }, + }, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess) } var props auth.MutableProps if err := xml.Unmarshal(ctx.Body(), &props); err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminUpdateUser, - }) + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) } err := props.Validate() if err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminUpdateUser, - }) + }, + }, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole) } err = c.iam.UpdateUserAccount(access, props) @@ -112,78 +115,97 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) error { err = s3err.GetAPIError(s3err.ErrAdminUserNotFound) } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminUpdateUser, - }) + }, + }, err } - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminUpdateUser, - }) + }, + }, nil } -func (c AdminController) DeleteUser(ctx *fiber.Ctx) error { +func (c AdminController) DeleteUser(ctx *fiber.Ctx) (*Response, error) { access := ctx.Query("access") err := c.iam.DeleteUserAccount(access) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminDeleteUser, - }) + }, + }, err } -func (c AdminController) ListUsers(ctx *fiber.Ctx) error { +func (c AdminController) ListUsers(ctx *fiber.Ctx) (*Response, error) { accs, err := c.iam.ListUserAccounts() - return SendXMLResponse(ctx, - auth.ListUserAccountsResult{ - Accounts: accs, - }, err, - &MetaOpts{ - Logger: c.l, + return &Response{ + Data: auth.ListUserAccountsResult{Accounts: accs}, + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminListUsers, - }) + }, + }, err } -func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error { +func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) (*Response, error) { owner := ctx.Query("owner") bucket := ctx.Query("bucket") accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam) if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminChangeBucketOwner, - }) + }, + }, err } if len(accs) > 0 { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound), - &MetaOpts{ - Logger: c.l, + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminChangeBucketOwner, - }) + }, + }, s3err.GetAPIError(s3err.ErrAdminUserNotFound) } - err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.l, + acl := auth.ACL{ + Owner: owner, + Grantees: []auth.Grantee{ + { + Permission: auth.PermissionFullControl, + Access: owner, + Type: types.TypeCanonicalUser, + }, + }, + } + + aclParsed, err := json.Marshal(acl) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionAdminChangeBucketOwner, + }, + }, fmt.Errorf("failed to marshal the bucket acl: %w", err) + } + + err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed) + return &Response{ + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminChangeBucketOwner, - }) + }, + }, err } -func (c AdminController) ListBuckets(ctx *fiber.Ctx) error { +func (c AdminController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { buckets, err := c.be.ListBucketsAndOwners(ctx.Context()) - return SendXMLResponse(ctx, - s3response.ListBucketsResult{ + return &Response{ + Data: s3response.ListBucketsResult{ Buckets: buckets, - }, err, &MetaOpts{ - Logger: c.l, + }, + MetaOpts: &MetaOptions{ Action: metrics.ActionAdminListBuckets, - }) + }, + }, err } diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go deleted file mode 100644 index aaa82a04..00000000 --- a/s3api/controllers/admin_test.go +++ /dev/null @@ -1,454 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package controllers - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/s3response" -) - -func TestAdminController_CreateUser(t *testing.T) { - type args struct { - req *http.Request - } - - adminController := AdminController{ - iam: &IAMServiceMock{ - CreateAccountFunc: func(account auth.Account) error { - return nil - }, - }, - } - - app := fiber.New() - - app.Patch("/create-user", adminController.CreateUser) - - succUser := ` - - access - secret - admin - 0 - 0 - - ` - invuser := ` - - access - secret - invalid_role - 0 - 0 - - ` - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Admin-create-user-malformed-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/create-user", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Admin-create-user-invalid-requester-role", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Admin-create-user-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)), - }, - wantErr: false, - statusCode: 201, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.CreateUser() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.CreateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestAdminController_UpdateUser(t *testing.T) { - type args struct { - req *http.Request - } - - adminController := AdminController{ - iam: &IAMServiceMock{ - UpdateUserAccountFunc: func(access string, props auth.MutableProps) error { - return nil - }, - }, - } - - app := fiber.New() - - app.Patch("/update-user", adminController.UpdateUser) - - adminControllerErr := AdminController{ - iam: &IAMServiceMock{ - UpdateUserAccountFunc: func(access string, props auth.MutableProps) error { - return auth.ErrNoSuchUser - }, - }, - } - - appNotFound := fiber.New() - - appNotFound.Patch("/update-user", adminControllerErr.UpdateUser) - - succUser := ` - - secret - 0 - 0 - - ` - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Admin-update-user-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Admin-update-user-missing-access", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)), - }, - wantErr: false, - statusCode: 404, - }, - { - name: "Admin-update-user-invalid-request-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Admin-update-user-not-found", - app: appNotFound, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)), - }, - wantErr: false, - statusCode: 404, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestAdminController_DeleteUser(t *testing.T) { - type args struct { - req *http.Request - } - - adminController := AdminController{ - iam: &IAMServiceMock{ - DeleteUserAccountFunc: func(access string) error { - return nil - }, - }, - } - - app := fiber.New() - - app.Patch("/delete-user", adminController.DeleteUser) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Admin-delete-user-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.DeleteUser() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.DeleteUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestAdminController_ListUsers(t *testing.T) { - type args struct { - req *http.Request - } - - adminController := AdminController{ - iam: &IAMServiceMock{ - ListUserAccountsFunc: func() ([]auth.Account, error) { - return []auth.Account{}, nil - }, - }, - } - - adminControllerErr := AdminController{ - iam: &IAMServiceMock{ - ListUserAccountsFunc: func() ([]auth.Account, error) { - return []auth.Account{}, fmt.Errorf("server error") - }, - }, - } - - appErr := fiber.New() - appErr.Patch("/list-users", adminControllerErr.ListUsers) - - appSucc := fiber.New() - appSucc.Patch("/list-users", adminController.ListUsers) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Admin-list-users-iam-error", - app: appErr, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/list-users", nil), - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Admin-list-users-success", - app: appSucc, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/list-users", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestAdminController_ChangeBucketOwner(t *testing.T) { - type args struct { - req *http.Request - } - adminController := AdminController{ - be: &BackendMock{ - ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, owner string) error { - return nil - }, - }, - iam: &IAMServiceMock{ - GetUserAccountFunc: func(access string) (auth.Account, error) { - return auth.Account{}, nil - }, - }, - } - - adminControllerIamErr := AdminController{ - iam: &IAMServiceMock{ - GetUserAccountFunc: func(access string) (auth.Account, error) { - return auth.Account{}, fmt.Errorf("unknown server error") - }, - }, - } - - adminControllerIamAccDoesNotExist := AdminController{ - iam: &IAMServiceMock{ - GetUserAccountFunc: func(access string) (auth.Account, error) { - return auth.Account{}, auth.ErrNoSuchUser - }, - }, - } - - app := fiber.New() - app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner) - - appIamErr := fiber.New() - appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner) - - appIamNoSuchUser := fiber.New() - appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Change-bucket-owner-check-account-server-error", - app: appIamErr, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil), - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Change-bucket-owner-acc-does-not-exist", - app: appIamNoSuchUser, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil), - }, - wantErr: false, - statusCode: 404, - }, - { - name: "Change-bucket-owner-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestAdminController_ListBuckets(t *testing.T) { - type args struct { - req *http.Request - } - adminController := AdminController{ - be: &BackendMock{ - ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) { - return []s3response.Bucket{}, nil - }, - }, - } - - app := fiber.New() - app.Patch("/list-buckets", adminController.ListBuckets) - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "List-buckets-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 34c36b46..00c79a17 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1991,93 +1991,3 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { ctx.Status(l.Status) return nil } - -func SendXMLResponse(ctx *fiber.Ctx, resp any, err error, l *MetaOpts) error { - if l.MetricsMng != nil { - if l.ObjectCount > 0 { - l.MetricsMng.Send(ctx, err, l.Action, l.ObjectCount, l.Status) - } else { - l.MetricsMng.Send(ctx, err, l.Action, l.ContentLength, l.Status) - } - } - if err != nil { - if l.Logger != nil { - l.Logger.Log(ctx, err, nil, s3log.LogMeta{ - Action: l.Action, - BucketOwner: l.BucketOwner, - ObjectSize: l.ObjectSize, - }) - } - serr, ok := err.(s3err.APIError) - if ok { - ctx.Status(serr.HTTPStatusCode) - return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", "")) - } - - fmt.Fprintf(os.Stderr, "Internal Error, %v\n", err) - ctx.Status(http.StatusInternalServerError) - - return ctx.Send(s3err.GetAPIErrorResponse( - s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) - } - - var b []byte - - // Handle already encoded responses(text, json...) - encodedResp, ok := resp.([]byte) - if ok { - b = encodedResp - } - - if resp != nil && !ok { - if b, err = xml.Marshal(resp); err != nil { - return err - } - - if len(b) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(b))) - ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) - } - } - - if l.Logger != nil { - l.Logger.Log(ctx, nil, b, s3log.LogMeta{ - Action: l.Action, - BucketOwner: l.BucketOwner, - ObjectSize: l.ObjectSize, - }) - } - - if l.EvSender != nil { - l.EvSender.SendEvent(ctx, s3event.EventMeta{ - BucketOwner: l.BucketOwner, - ObjectSize: l.ObjectSize, - ObjectETag: l.ObjectETag, - VersionId: l.VersionId, - EventName: l.EventName, - }) - } - - if ok { - if len(b) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(b))) - } - - return ctx.Send(b) - } - - msglen := len(xmlhdr) + len(b) - if msglen > maxXMLBodyLen { - debuglogger.Logf("XML encoded body len %v exceeds max len %v", - msglen, maxXMLBodyLen) - ctx.Status(http.StatusInternalServerError) - - return ctx.Send(s3err.GetAPIErrorResponse( - s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) - } - res := make([]byte, 0, msglen) - res = append(res, xmlhdr...) - res = append(res, b...) - - return ctx.Send(res) -} diff --git a/s3api/middlewares/acl-parser.go b/s3api/middlewares/acl-parser.go index 20a4a56f..8a8dceed 100644 --- a/s3api/middlewares/acl-parser.go +++ b/s3api/middlewares/acl-parser.go @@ -55,10 +55,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe !ctx.Request().URI().QueryArgs().Has("cors") { isRoot, acct := utils.ContextKeyIsRoot.Get(ctx).(bool), utils.ContextKeyAccount.Get(ctx).(auth.Account) if err := auth.MayCreateBucket(acct, isRoot); err != nil { - return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"}) + return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"}) } if readonly { - return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied), + return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAccessDenied), &controllers.MetaOpts{ Logger: logger, Action: "CreateBucket", diff --git a/s3api/router.go b/s3api/router.go index 31aa0ced..047d35aa 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -36,22 +36,22 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ adminController := controllers.NewAdminController(iam, be, aLogger) // CreateUser admin api - app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser) + app.Patch("/create-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.CreateUser, aLogger, nil, nil)) // DeleteUsers admin api - app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser) + app.Patch("/delete-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.DeleteUser, aLogger, nil, nil)) // UpdateUser admin api - app.Patch("/update-user", middlewares.IsAdmin(logger), adminController.UpdateUser) + app.Patch("/update-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.UpdateUser, aLogger, nil, nil)) // ListUsers admin api - app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers) + app.Patch("/list-users", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ListUsers, aLogger, nil, nil)) // ChangeBucketOwner admin api - app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner) + app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ChangeBucketOwner, aLogger, nil, nil)) // ListBucketsAndOwners admin api - app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets) + app.Patch("/list-buckets", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ListBuckets, aLogger, nil, nil)) } // ListBuckets action From 5be9e3bd1e5c115b08f0da2197c9d495e75750c9 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sat, 28 Jun 2025 02:50:14 +0400 Subject: [PATCH 11/20] feat: a total refactoring of the gateway middlewares by lowering them from server to router level. --- auth/access-control.go | 12 - s3api/admin-router.go | 43 +- s3api/admin-server.go | 12 +- s3api/controllers/admin.go | 64 +-- s3api/controllers/base.go | 223 +++++++-- s3api/controllers/bucket-delete.go | 11 - s3api/controllers/bucket-get.go | 31 -- s3api/controllers/bucket-head.go | 4 - s3api/controllers/bucket-list.go | 11 +- s3api/controllers/bucket-post.go | 5 - s3api/controllers/bucket-put.go | 50 +- s3api/controllers/object-delete.go | 11 +- s3api/controllers/object-get.go | 25 - s3api/controllers/object-head.go | 6 - s3api/controllers/object-post.go | 19 - s3api/controllers/object-put.go | 41 -- s3api/controllers/utilities.go | 195 -------- s3api/middlewares/acl-parser.go | 53 +-- s3api/middlewares/admin.go | 34 +- s3api/middlewares/authentication.go | 53 +-- s3api/middlewares/md5.go | 14 +- s3api/middlewares/presign-auth.go | 26 +- s3api/middlewares/public-bucket.go | 4 +- s3api/middlewares/url-decoder.go | 20 +- s3api/router.go | 690 +++++++++++++++++++++++++--- s3api/router_test.go | 3 +- s3api/server.go | 6 +- tests/integration/s3conf.go | 4 + tests/integration/tests.go | 150 +++--- tests/integration/utils.go | 18 +- 30 files changed, 1034 insertions(+), 804 deletions(-) delete mode 100644 s3api/controllers/utilities.go diff --git a/auth/access-control.go b/auth/access-control.go index d984c9d4..a726a4ce 100644 --- a/auth/access-control.go +++ b/auth/access-control.go @@ -155,18 +155,6 @@ func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, return nil } -func MayCreateBucket(acct Account, isRoot bool) error { - if isRoot { - return nil - } - - if acct.Role == RoleUser { - return s3err.GetAPIError(s3err.ErrAccessDenied) - } - - return nil -} - func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error { // Owner check if acct.Access == acl.Owner { diff --git a/s3api/admin-router.go b/s3api/admin-router.go index 16c2df52..ca10b639 100644 --- a/s3api/admin-router.go +++ b/s3api/admin-router.go @@ -18,30 +18,59 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/controllers" + "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3log" ) type S3AdminRouter struct{} -func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) { +func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, root middlewares.RootUserConfig, region string, debug bool) { ctrl := controllers.NewAdminController(iam, be, logger) + services := &controllers.Services{ + Logger: logger, + } // CreateUser admin api - app.Patch("/create-user", controllers.ProcessResponse(ctrl.CreateUser, logger, nil, nil)) + app.Patch("/create-user", + controllers.ProcessHandlers(ctrl.CreateUser, metrics.ActionAdminCreateUser, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminCreateUser), + )) // DeleteUsers admin api - app.Patch("/delete-user", controllers.ProcessResponse(ctrl.DeleteUser, logger, nil, nil)) + app.Patch("/delete-user", + controllers.ProcessHandlers(ctrl.DeleteUser, metrics.ActionAdminDeleteUser, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminDeleteUser), + )) // UpdateUser admin api - app.Patch("/update-user", controllers.ProcessResponse(ctrl.UpdateUser, logger, nil, nil)) + app.Patch("/update-user", + controllers.ProcessHandlers(ctrl.UpdateUser, metrics.ActionAdminUpdateUser, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminUpdateUser), + )) // ListUsers admin api - app.Patch("/list-users", controllers.ProcessResponse(ctrl.ListUsers, logger, nil, nil)) + app.Patch("/list-users", + controllers.ProcessHandlers(ctrl.ListUsers, metrics.ActionAdminListUsers, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminListUsers), + )) // ChangeBucketOwner admin api - app.Patch("/change-bucket-owner", controllers.ProcessResponse(ctrl.ChangeBucketOwner, logger, nil, nil)) + app.Patch("/change-bucket-owner", + controllers.ProcessHandlers(ctrl.ChangeBucketOwner, metrics.ActionAdminChangeBucketOwner, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminChangeBucketOwner), + )) // ListBucketsAndOwners admin api - app.Patch("/list-buckets", controllers.ProcessResponse(ctrl.ListBuckets, logger, nil, nil)) + app.Patch("/list-buckets", + controllers.ProcessHandlers(ctrl.ListBuckets, metrics.ActionAdminListBuckets, services, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminListBuckets), + )) } diff --git a/s3api/admin-server.go b/s3api/admin-server.go index 23b66019..979f4793 100644 --- a/s3api/admin-server.go +++ b/s3api/admin-server.go @@ -21,6 +21,7 @@ import ( "github.com/gofiber/fiber/v2/middleware/logger" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3log" ) @@ -53,17 +54,10 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n", })) } - app.Use(middlewares.DecodeURL(l, nil)) + app.Use(controllers.WrapMiddleware(middlewares.DecodeURL, l, nil)) app.Use(middlewares.DebugLogger()) - // Authentication middlewares - app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false)) - app.Use(middlewares.VerifyMD5Body(l)) - - // Admin role checker - app.Use(middlewares.IsAdmin(l)) - - server.router.Init(app, be, iam, l) + server.router.Init(app, be, iam, l, root, region, server.debug) return server } diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index 30cb6710..cbf19427 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -25,7 +25,6 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3log" "github.com/versity/versitygw/s3response" @@ -46,17 +45,13 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) (*Response, error) { err := xml.Unmarshal(ctx.Body(), &usr) if err != nil { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminCreateUser, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrMalformedXML) } if !usr.Role.IsValid() { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminCreateUser, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole) } @@ -67,15 +62,12 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) (*Response, error) { } return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminCreateUser, - }, + MetaOpts: &MetaOptions{}, }, err } return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminCreateUser, Status: http.StatusCreated, }, }, nil @@ -85,27 +77,21 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) (*Response, error) { access := ctx.Query("access") if access == "" { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminUpdateUser, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess) } var props auth.MutableProps if err := xml.Unmarshal(ctx.Body(), &props); err != nil { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminUpdateUser, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrMalformedXML) } err := props.Validate() if err != nil { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminUpdateUser, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole) } @@ -116,16 +102,12 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) (*Response, error) { } return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminUpdateUser, - }, + MetaOpts: &MetaOptions{}, }, err } return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminUpdateUser, - }, + MetaOpts: &MetaOptions{}, }, nil } @@ -134,19 +116,15 @@ func (c AdminController) DeleteUser(ctx *fiber.Ctx) (*Response, error) { err := c.iam.DeleteUserAccount(access) return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminDeleteUser, - }, + MetaOpts: &MetaOptions{}, }, err } func (c AdminController) ListUsers(ctx *fiber.Ctx) (*Response, error) { accs, err := c.iam.ListUserAccounts() return &Response{ - Data: auth.ListUserAccountsResult{Accounts: accs}, - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminListUsers, - }, + Data: auth.ListUserAccountsResult{Accounts: accs}, + MetaOpts: &MetaOptions{}, }, err } @@ -157,16 +135,12 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) (*Response, error) { accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam) if err != nil { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminChangeBucketOwner, - }, + MetaOpts: &MetaOptions{}, }, err } if len(accs) > 0 { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminChangeBucketOwner, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrAdminUserNotFound) } @@ -184,17 +158,13 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) (*Response, error) { aclParsed, err := json.Marshal(acl) if err != nil { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminChangeBucketOwner, - }, + MetaOpts: &MetaOptions{}, }, fmt.Errorf("failed to marshal the bucket acl: %w", err) } err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed) return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminChangeBucketOwner, - }, + MetaOpts: &MetaOptions{}, }, err } @@ -204,8 +174,6 @@ func (c AdminController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { Data: s3response.ListBucketsResult{ Buckets: buckets, }, - MetaOpts: &MetaOptions{ - Action: metrics.ActionAdminListBuckets, - }, + MetaOpts: &MetaOptions{}, }, err } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 00c79a17..15713d2c 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -53,7 +53,12 @@ type S3ApiController struct { const ( iso8601Format = "20060102T150405Z" iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006" - defaultContentType = "binary/octet-stream" + timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" + maxXMLBodyLen = 4 * 1024 * 1024 +) + +var ( + xmlhdr = []byte(`` + "\n") ) func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController { @@ -72,6 +77,11 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } +// Returns MethodNotAllowed for unmatched routes +func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) +} + func getint64(i *int64) int64 { if i == nil { return 0 @@ -1936,7 +1946,6 @@ type MetaOpts struct { EvSender s3event.S3EventSender MetricsMng *metrics.Manager ContentLength int64 - Action string BucketOwner string ObjectSize int64 ObjectCount int64 @@ -1946,48 +1955,202 @@ type MetaOpts struct { Status int } -func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { - if l.Logger != nil { - l.Logger.Log(ctx, err, nil, s3log.LogMeta{ - Action: l.Action, - BucketOwner: l.BucketOwner, - ObjectSize: l.ObjectSize, - }) +// Response is the type definition for a controller response +// Data - Response body +// Headers - Resposne headers +// MetaOpts - Meta options for metrics, audit logs and s3 events +type Response struct { + Data any + Headers map[string]*string + MetaOpts *MetaOptions +} + +// Services groups the metrics manager, s3 event sender and audit logger +type Services struct { + Logger s3log.AuditLogger + EventSender s3event.S3EventSender + MetricsManager *metrics.Manager +} + +// Controller is the type definition for an s3api controller +type Controller func(ctx *fiber.Ctx) (*Response, error) + +// ProcessHandlers groups a controller and multiple middlewares into a single fiber handler +func ProcessHandlers(controller Controller, s3action string, svc *Services, handlers ...fiber.Handler) fiber.Handler { + return func(ctx *fiber.Ctx) error { + // if skip locals is set, skip to the next rout handler + if utils.ContextKeySkip.IsSet(ctx) { + utils.ContextKeySkip.Delete(ctx) + return ctx.Next() + } + + for _, handler := range handlers { + err := handler(ctx) + if err != nil { + return ProcessController(ctx, func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{}, + }, err + }, s3action, svc) + } + } + + return ProcessController(ctx, controller, s3action, svc) } - if l.MetricsMng != nil { - if l.ObjectCount > 0 { - l.MetricsMng.Send(ctx, err, l.Action, l.ObjectCount, l.Status) +} + +// WrapMiddleware executes the given middleware and handles sending the audit logs +// and metrics. It also handles the error parsing +func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { + return func(ctx *fiber.Ctx) error { + err := handler(ctx) + if mm != nil { + mm.Send(ctx, err, metrics.ActionUndetected, 0, 0) + } + if logger != nil { + logger.Log(ctx, err, ctx.Body(), s3log.LogMeta{ + Action: metrics.ActionUndetected, + }) + } + + if err != nil { + serr, ok := err.(s3err.APIError) + if ok { + ctx.Status(serr.HTTPStatusCode) + return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", "")) + } + + debuglogger.Logf("Internal Error, %v", err) + ctx.Status(http.StatusInternalServerError) + + // If the error is not 's3err.APIError' return 'InternalError' + return ctx.Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) + } + + return ctx.Next() + } +} + +// ProcessController executes the given s3api controller and handles the metrics +// access logs and s3 events +func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, svc *Services) error { + response, err := controller(ctx) + + // Set the response headers + SetResponseHeaders(ctx, response.Headers) + + opts := response.MetaOpts + // Send the metrics + if svc.MetricsManager != nil { + if opts.ObjectCount > 0 { + svc.MetricsManager.Send(ctx, err, s3action, opts.ObjectCount, opts.Status) } else { - l.MetricsMng.Send(ctx, err, l.Action, l.ContentLength, l.Status) + svc.MetricsManager.Send(ctx, err, s3action, opts.ContentLength, opts.Status) } } + // Handle the error case if err != nil { - var apierr s3err.APIError - if errors.As(err, &apierr) { - ctx.Status(apierr.HTTPStatusCode) - return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", "")) + // Audit the error log + if svc.Logger != nil { + svc.Logger.Log(ctx, err, nil, s3log.LogMeta{ + Action: s3action, + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, + }) + } + serr, ok := err.(s3err.APIError) + if ok { + ctx.Status(serr.HTTPStatusCode) + return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", "")) } fmt.Fprintf(os.Stderr, "Internal Error, %v\n", err) ctx.Status(http.StatusInternalServerError) + + // If the error is not 's3err.APIError' return 'InternalError' return ctx.Send(s3err.GetAPIErrorResponse( s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) } - if l.EvSender != nil { - l.EvSender.SendEvent(ctx, s3event.EventMeta{ - ObjectSize: l.ObjectSize, - ObjectETag: l.ObjectETag, - EventName: l.EventName, - BucketOwner: l.BucketOwner, - VersionId: l.VersionId, + + if opts.Status == 0 { + opts.Status = http.StatusOK + } + + // if no data payload is provided, send the response status + if response.Data == nil { + ctx.Status(opts.Status) + return nil + } + + var responseBytes []byte + + // Handle already encoded responses(text, json...) + encodedResp, ok := response.Data.([]byte) + if ok { + responseBytes = encodedResp + } else { + if responseBytes, err = xml.Marshal(response.Data); err != nil { + return err + } + + if len(responseBytes) > 0 { + ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) + ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) + } + } + + if svc.Logger != nil { + svc.Logger.Log(ctx, nil, responseBytes, s3log.LogMeta{ + Action: s3action, + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, }) } - if l.Status == 0 { - l.Status = http.StatusOK + if svc.EventSender != nil { + svc.EventSender.SendEvent(ctx, s3event.EventMeta{ + BucketOwner: opts.BucketOwner, + ObjectSize: opts.ObjectSize, + ObjectETag: opts.ObjectETag, + VersionId: opts.VersionId, + EventName: opts.EventName, + }) + } + + if ok { + if len(responseBytes) > 0 { + ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) + } + + return ctx.Send(responseBytes) + } + + msglen := len(xmlhdr) + len(responseBytes) + if msglen > maxXMLBodyLen { + debuglogger.Logf("XML encoded body len %v exceeds max len %v", + msglen, maxXMLBodyLen) + ctx.Status(http.StatusInternalServerError) + + return ctx.Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) + } + res := make([]byte, 0, msglen) + res = append(res, xmlhdr...) + res = append(res, responseBytes...) + + return ctx.Send(res) +} + +// Sets the response headers +func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { + if headers == nil { + return + } + for key, val := range headers { + if val == nil || *val == "" { + continue + } + ctx.Response().Header.Add(key, *val) } - // https://github.com/gofiber/fiber/issues/2080 - // ctx.SendStatus() sets incorrect content length on HEAD request - ctx.Status(l.Status) - return nil } diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index 59a49f62..00a169dd 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -19,7 +19,6 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" ) @@ -44,7 +43,6 @@ func (c S3ApiController) DeleteBucketTagging(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -53,7 +51,6 @@ func (c S3ApiController) DeleteBucketTagging(ctx *fiber.Ctx) (*Response, error) err = c.be.DeleteBucketTagging(ctx.Context(), bucket) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketTagging, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, @@ -79,7 +76,6 @@ func (c S3ApiController) DeleteBucketOwnershipControls(ctx *fiber.Ctx) (*Respons if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, err @@ -88,7 +84,6 @@ func (c S3ApiController) DeleteBucketOwnershipControls(ctx *fiber.Ctx) (*Respons err = c.be.DeleteBucketOwnershipControls(ctx.Context(), bucket) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketOwnershipControls, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, @@ -114,7 +109,6 @@ func (c S3ApiController) DeleteBucketPolicy(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -123,7 +117,6 @@ func (c S3ApiController) DeleteBucketPolicy(ctx *fiber.Ctx) (*Response, error) { err = c.be.DeleteBucketPolicy(ctx.Context(), bucket) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketPolicy, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, @@ -151,7 +144,6 @@ func (c S3ApiController) DeleteBucketCors(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -160,7 +152,6 @@ func (c S3ApiController) DeleteBucketCors(ctx *fiber.Ctx) (*Response, error) { err = c.be.DeleteBucketCors(ctx.Context(), bucket) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -187,7 +178,6 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucket, BucketOwner: parsedAcl.Owner, }, }, err @@ -196,7 +186,6 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) (*Response, error) { err = c.be.DeleteBucket(ctx.Context(), bucket) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteBucket, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go index d115519b..cc60d267 100644 --- a/s3api/controllers/bucket-get.go +++ b/s3api/controllers/bucket-get.go @@ -21,7 +21,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -48,7 +47,6 @@ func (c S3ApiController) GetBucketTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -58,7 +56,6 @@ func (c S3ApiController) GetBucketTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -75,7 +72,6 @@ func (c S3ApiController) GetBucketTagging(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: resp, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -101,7 +97,6 @@ func (c S3ApiController) GetBucketOwnershipControls(ctx *fiber.Ctx) (*Response, if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, err @@ -117,7 +112,6 @@ func (c S3ApiController) GetBucketOwnershipControls(ctx *fiber.Ctx) (*Response, }, }, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, err @@ -143,7 +137,6 @@ func (c S3ApiController) GetBucketVersioning(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, err @@ -152,7 +145,6 @@ func (c S3ApiController) GetBucketVersioning(ctx *fiber.Ctx) (*Response, error) if err := auth.IsAdminOrOwner(acct, isRoot, parsedAcl); err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, err @@ -162,7 +154,6 @@ func (c S3ApiController) GetBucketVersioning(ctx *fiber.Ctx) (*Response, error) return &Response{ Data: data, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, err @@ -188,7 +179,6 @@ func (c S3ApiController) GetBucketCors(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -198,7 +188,6 @@ func (c S3ApiController) GetBucketCors(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: data, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -224,7 +213,6 @@ func (c S3ApiController) GetBucketPolicy(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -234,7 +222,6 @@ func (c S3ApiController) GetBucketPolicy(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: data, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -267,7 +254,6 @@ func (c S3ApiController) ListObjectVersions(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectVersions, BucketOwner: parsedAcl.Owner, }, }, err @@ -279,7 +265,6 @@ func (c S3ApiController) ListObjectVersions(ctx *fiber.Ctx) (*Response, error) { maxkeysStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectVersions, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) @@ -297,7 +282,6 @@ func (c S3ApiController) ListObjectVersions(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: data, MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectVersions, BucketOwner: parsedAcl.Owner, }, }, err @@ -325,7 +309,6 @@ func (c S3ApiController) GetObjectLockConfiguration(ctx *fiber.Ctx) (*Response, if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -335,7 +318,6 @@ func (c S3ApiController) GetObjectLockConfiguration(ctx *fiber.Ctx) (*Response, if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -345,7 +327,6 @@ func (c S3ApiController) GetObjectLockConfiguration(ctx *fiber.Ctx) (*Response, return &Response{ Data: resp, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -373,7 +354,6 @@ func (c S3ApiController) GetBucketAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -384,7 +364,6 @@ func (c S3ApiController) GetBucketAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -394,7 +373,6 @@ func (c S3ApiController) GetBucketAcl(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -427,7 +405,6 @@ func (c S3ApiController) ListMultipartUploads(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListMultipartUploads, BucketOwner: parsedAcl.Owner, }, }, err @@ -438,7 +415,6 @@ func (c S3ApiController) ListMultipartUploads(ctx *fiber.Ctx) (*Response, error) maxUploadsStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListMultipartUploads, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxUploads) @@ -455,7 +431,6 @@ func (c S3ApiController) ListMultipartUploads(ctx *fiber.Ctx) (*Response, error) return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionListMultipartUploads, BucketOwner: parsedAcl.Owner, }, }, err @@ -489,7 +464,6 @@ func (c S3ApiController) ListObjectsV2(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectsV2, BucketOwner: parsedAcl.Owner, }, }, err @@ -500,7 +474,6 @@ func (c S3ApiController) ListObjectsV2(ctx *fiber.Ctx) (*Response, error) { maxkeysStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectsV2, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) @@ -519,7 +492,6 @@ func (c S3ApiController) ListObjectsV2(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjectsV2, BucketOwner: parsedAcl.Owner, }, }, err @@ -551,7 +523,6 @@ func (c S3ApiController) ListObjects(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjects, BucketOwner: parsedAcl.Owner, }, }, err @@ -563,7 +534,6 @@ func (c S3ApiController) ListObjects(ctx *fiber.Ctx) (*Response, error) { maxkeysStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjects, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxKeys) @@ -580,7 +550,6 @@ func (c S3ApiController) ListObjects(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionListObjects, BucketOwner: parsedAcl.Owner, }, }, err diff --git a/s3api/controllers/bucket-head.go b/s3api/controllers/bucket-head.go index bac27962..f4f063dc 100644 --- a/s3api/controllers/bucket-head.go +++ b/s3api/controllers/bucket-head.go @@ -18,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" ) @@ -44,7 +43,6 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadBucket, BucketOwner: parsedAcl.Owner, }, }, err @@ -58,7 +56,6 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadBucket, BucketOwner: parsedAcl.Owner, }, }, err @@ -70,7 +67,6 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { "X-Amz-Bucket-Region": utils.GetStringPtr(region), }, MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadBucket, BucketOwner: parsedAcl.Owner, }, }, nil diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go index c2ceaa9c..38bcb327 100644 --- a/s3api/controllers/bucket-list.go +++ b/s3api/controllers/bucket-list.go @@ -19,7 +19,6 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -39,9 +38,7 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > 10000 { debuglogger.Logf("error parsing max-buckets %q: %v", maxBucketsStr, err) return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionListAllMyBuckets, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrInvalidMaxBuckets) } maxBuckets = int32(maxBucketsParsed) @@ -56,9 +53,7 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { Prefix: prefix, }) return &Response{ - Data: res, - MetaOpts: &MetaOptions{ - Action: metrics.ActionListAllMyBuckets, - }, + Data: res, + MetaOpts: &MetaOptions{}, }, err } diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index f9ce29cd..283afc7f 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -22,7 +22,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -45,7 +44,6 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error unmarshalling delete objects: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObjects, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -65,7 +63,6 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObjects, BucketOwner: parsedAcl.Owner, }, }, err @@ -75,7 +72,6 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObjects, BucketOwner: parsedAcl.Owner, }, }, err @@ -91,7 +87,6 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObjects, ObjectCount: int64(len(dObj.Objects)), BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectRemovedDeleteObjects, diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index 5fa73c3f..6080c06b 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -25,7 +25,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -52,7 +51,6 @@ func (c S3ApiController) PutBucketTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -62,7 +60,6 @@ func (c S3ApiController) PutBucketTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -71,7 +68,6 @@ func (c S3ApiController) PutBucketTagging(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutBucketTagging(ctx.Context(), bucket, tagging) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketTagging, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, @@ -95,7 +91,6 @@ func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, }); err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, err @@ -106,7 +101,6 @@ func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, debuglogger.Logf("failed to unmarshal request body: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -120,7 +114,6 @@ func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, } return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -129,7 +122,6 @@ func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketOwnershipControls, BucketOwner: parsedAcl.Owner, }, }, err @@ -155,7 +147,6 @@ func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, err @@ -167,7 +158,6 @@ func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) debuglogger.Logf("error unmarshalling versioning configuration: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -178,7 +168,6 @@ func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) debuglogger.Logf("invalid versioning configuration status: %v", versioningConf.Status) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -187,7 +176,6 @@ func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketVersioning, BucketOwner: parsedAcl.Owner, }, }, err @@ -212,7 +200,6 @@ func (c S3ApiController) PutObjectLockConfiguration(ctx *fiber.Ctx) (*Response, }); err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -222,7 +209,6 @@ func (c S3ApiController) PutObjectLockConfiguration(ctx *fiber.Ctx) (*Response, if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -231,7 +217,6 @@ func (c S3ApiController) PutObjectLockConfiguration(ctx *fiber.Ctx) (*Response, err = c.be.PutObjectLockConfiguration(ctx.Context(), bucket, config) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLockConfiguration, BucketOwner: parsedAcl.Owner, }, }, err @@ -257,7 +242,6 @@ func (c S3ApiController) PutBucketCors(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -266,7 +250,6 @@ func (c S3ApiController) PutBucketCors(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutBucketCors(ctx.Context(), []byte{}) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketCors, BucketOwner: parsedAcl.Owner, }, }, err @@ -290,7 +273,6 @@ func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -300,7 +282,6 @@ func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -309,7 +290,6 @@ func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body()) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketPolicy, BucketOwner: parsedAcl.Owner, }, }, err @@ -335,7 +315,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -344,7 +323,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("bucket acls are disabled") return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrAclNotSupported) @@ -363,7 +341,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -376,7 +353,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error unmarshalling access control policy: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedACL) @@ -387,7 +363,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid access control policy: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -397,7 +372,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.APIError{ @@ -412,7 +386,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { grants, acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrUnexpectedContent) @@ -427,7 +400,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid acl: %q", acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -437,7 +409,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { grants, acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) @@ -460,7 +431,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("none of the bucket acl options has been specified: canned, req headers, req body") return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMissingSecurityHeader) @@ -470,7 +440,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -479,7 +448,6 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutBucketAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -500,21 +468,23 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), ) + if acct.Role != auth.RoleAdmin && acct.Role != auth.RoleUserPlus { + return &Response{ + MetaOpts: &MetaOptions{}, + }, s3err.GetAPIError(s3err.ErrAccessDenied) + } + // validate the bucket name if ok := utils.IsValidBucketName(bucket); !ok { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, - }, + MetaOpts: &MetaOptions{}, }, s3err.GetAPIError(s3err.ErrInvalidBucketName) } // validate the object ownership value if ok := utils.IsValidOwnership(objectOwnership); !ok { return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, - }, + MetaOpts: &MetaOptions{}, }, s3err.APIError{ Code: "InvalidArgument", Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership), @@ -526,7 +496,6 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("bucket acls are disabled for %v object ownership", objectOwnership) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, BucketOwner: acct.Access, }, }, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership) @@ -536,7 +505,6 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, BucketOwner: acct.Access, }, }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) @@ -562,7 +530,6 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("failed to update bucket acl: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, BucketOwner: acct.Access, }, }, err @@ -575,7 +542,6 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { }, updAcl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateBucket, BucketOwner: acct.Access, }, }, err diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go index 2cbebae2..93721315 100644 --- a/s3api/controllers/object-delete.go +++ b/s3api/controllers/object-delete.go @@ -23,7 +23,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3event" ) @@ -51,7 +50,6 @@ func (c S3ApiController) DeleteObjectTagging(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -61,14 +59,13 @@ func (c S3ApiController) DeleteObjectTagging(ctx *fiber.Ctx) (*Response, error) return &Response{ MetaOpts: &MetaOptions{ Status: http.StatusNoContent, - Action: metrics.ActionDeleteObjectTagging, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectTaggingDelete, }, }, err } -func (c S3ApiController) AbortMultipartUplaod(ctx *fiber.Ctx) (*Response, error) { +func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) uploadId := ctx.Query("uploadId") @@ -92,7 +89,6 @@ func (c S3ApiController) AbortMultipartUplaod(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionAbortMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -106,7 +102,6 @@ func (c S3ApiController) AbortMultipartUplaod(ctx *fiber.Ctx) (*Response, error) }) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionAbortMultipartUpload, BucketOwner: parsedAcl.Owner, Status: http.StatusNoContent, }, @@ -141,7 +136,6 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -164,7 +158,6 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -179,7 +172,6 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObject, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectRemovedDelete, Status: http.StatusNoContent, @@ -198,7 +190,6 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { return &Response{ Headers: headers, MetaOpts: &MetaOptions{ - Action: metrics.ActionDeleteObject, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectRemovedDelete, Status: http.StatusNoContent, diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index e8f16b0d..98d97917 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -26,7 +26,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -55,7 +54,6 @@ func (c S3ApiController) GetObjectTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -65,7 +63,6 @@ func (c S3ApiController) GetObjectTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -82,7 +79,6 @@ func (c S3ApiController) GetObjectTagging(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: tags, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -112,7 +108,6 @@ func (c S3ApiController) GetObjectRetention(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -122,7 +117,6 @@ func (c S3ApiController) GetObjectRetention(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -132,7 +126,6 @@ func (c S3ApiController) GetObjectRetention(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: retention, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -162,7 +155,6 @@ func (c S3ApiController) GetObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, err @@ -172,7 +164,6 @@ func (c S3ApiController) GetObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: auth.ParseObjectLegalHoldOutput(data), MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, err @@ -201,7 +192,6 @@ func (c S3ApiController) GetObjectAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -213,7 +203,6 @@ func (c S3ApiController) GetObjectAcl(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -240,7 +229,6 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListParts, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker) @@ -254,7 +242,6 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { maxPartsStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListParts, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxParts) @@ -274,7 +261,6 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionListParts, BucketOwner: parsedAcl.Owner, }, }, err @@ -290,7 +276,6 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { return &Response{ Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionListParts, BucketOwner: parsedAcl.Owner, }, }, err @@ -322,7 +307,6 @@ func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, err @@ -335,7 +319,6 @@ func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) maxPartsStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMaxParts) @@ -346,7 +329,6 @@ func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, err @@ -370,7 +352,6 @@ func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) return &Response{ Headers: headers, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, err @@ -388,7 +369,6 @@ func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) Headers: headers, Data: utils.FilterObjectAttributes(attrs, res), MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, err @@ -426,7 +406,6 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -437,7 +416,6 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") @@ -461,7 +439,6 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { return &Response{ Headers: headers, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObjectAttributes, BucketOwner: parsedAcl.Owner, }, }, err @@ -484,7 +461,6 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { *res.ContentLength) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObject, ContentLength: getint64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, @@ -525,7 +501,6 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), }, MetaOpts: &MetaOptions{ - Action: metrics.ActionGetObject, ContentLength: getint64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index 360eecca..2eb84d46 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -23,7 +23,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -48,7 +47,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid part number: %d", partNumberQuery) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) @@ -72,7 +70,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -83,7 +80,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") @@ -109,7 +105,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { return &Response{ Headers: headers, MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -146,7 +141,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), }, MetaOpts: &MetaOptions{ - Action: metrics.ActionHeadObject, BucketOwner: parsedAcl.Owner, }, }, nil diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index bbe96643..0d6fe8cb 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -24,7 +24,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -45,7 +44,6 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("failed to parse the request body: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionRestoreObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -65,7 +63,6 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionRestoreObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -78,7 +75,6 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { }) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionRestoreObject, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectRestoreCompleted, }, @@ -100,7 +96,6 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) debuglogger.Logf("error unmarshalling select object content: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionSelectObjectContent, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -121,7 +116,6 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionSelectObjectContent, BucketOwner: parsedAcl.Owner, }, }, err @@ -143,7 +137,6 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionSelectObjectContent, BucketOwner: parsedAcl.Owner, }, }, nil @@ -179,7 +172,6 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -189,7 +181,6 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -199,7 +190,6 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -233,7 +223,6 @@ func (c S3ApiController) CreateMultipartUpload(ctx *fiber.Ctx) (*Response, error Headers: headers, Data: res, MetaOpts: &MetaOptions{ - Action: metrics.ActionCreateMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -266,7 +255,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -281,7 +269,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err debuglogger.Logf("error unmarshalling complete multipart upload: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -291,7 +278,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err debuglogger.Logf("empty parts provided for complete multipart upload") return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrEmptyParts) @@ -305,7 +291,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err debuglogger.Logf("invalid value for 'x-amz-mp-objects-size' header: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -315,7 +300,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err debuglogger.Logf("value for 'x-amz-mp-objects-size' header is less than 0: %v", val) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, s3err.GetInvalidMpObjectSizeErr(val) @@ -328,7 +312,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -338,7 +321,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, }, }, err @@ -366,7 +348,6 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err "x-amz-version-id": &versid, }, MetaOpts: &MetaOptions{ - Action: metrics.ActionCompleteMultipartUpload, BucketOwner: parsedAcl.Owner, ObjectETag: res.ETag, EventName: s3event.EventCompleteMultipartUpload, diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 3e1de9fa..453f624f 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -27,7 +27,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/debuglogger" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" @@ -47,7 +46,6 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -67,7 +65,6 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectTagging, BucketOwner: parsedAcl.Owner, }, }, err @@ -76,7 +73,6 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutObjectTagging(ctx.Context(), bucket, key, tagging) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectTagging, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectTaggingPut, }, @@ -106,7 +102,6 @@ func (c S3ApiController) PutObjectRetention(ctx *fiber.Ctx) (*Response, error) { }); err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -128,7 +123,6 @@ func (c S3ApiController) PutObjectRetention(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("failed to parse object lock configuration input: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -137,7 +131,6 @@ func (c S3ApiController) PutObjectRetention(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutObjectRetention(ctx.Context(), bucket, key, versionId, bypass, retention) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectRetention, BucketOwner: parsedAcl.Owner, }, }, err @@ -157,7 +150,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("failed to parse request body: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -167,7 +159,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid legal hold status: %v", legalHold.Status) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrMalformedXML) @@ -186,7 +177,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { }); err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, err @@ -195,7 +185,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { err := c.be.PutObjectLegalHold(ctx.Context(), bucket, key, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectLegalHold, BucketOwner: parsedAcl.Owner, }, }, err @@ -227,7 +216,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid part number: %d", partNumber) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPart, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) @@ -248,7 +236,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPart, BucketOwner: parsedAcl.Owner, }, }, err @@ -259,7 +246,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPart, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -270,7 +256,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("err parsing checksum headers: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPart, BucketOwner: parsedAcl.Owner, }, }, err @@ -314,7 +299,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { Headers: headers, MetaOpts: &MetaOptions{ ContentLength: contentLength, - Action: metrics.ActionUploadPart, BucketOwner: parsedAcl.Owner, }, }, err @@ -340,7 +324,6 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error unescaping copy source %q: %v", cs, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPartCopy, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -350,7 +333,6 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid part number: %d", partNumber) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPartCopy, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) @@ -370,7 +352,6 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPartCopy, BucketOwner: parsedAcl.Owner, }, }, err @@ -395,7 +376,6 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { Headers: headers, Data: resp, MetaOpts: &MetaOptions{ - Action: metrics.ActionUploadPartCopy, BucketOwner: parsedAcl.Owner, }, }, err @@ -430,7 +410,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, }, }, err @@ -442,7 +421,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -454,7 +432,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error unmarshalling access control policy: %v", err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -488,7 +465,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid acl: %q", acl) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -497,7 +473,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", grants, acl, len(ctx.Body())) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -531,7 +506,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { err = c.be.PutObjectAcl(ctx.Context(), input) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObjectAcl, BucketOwner: parsedAcl.Owner, EventName: s3event.EventObjectAclPut, }, @@ -567,7 +541,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error unescaping copy source %q: %v", cs, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -586,7 +559,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -599,7 +571,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -613,7 +584,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -627,7 +597,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid metadata directive: %v", metaDirective) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidMetadataDirective) @@ -637,7 +606,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("invalid tagging direcrive: %v", taggingDirective) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidTaggingDirective) @@ -648,7 +616,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -658,7 +625,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -703,7 +669,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, Data: res.CopyObjectResult, MetaOpts: &MetaOptions{ - Action: metrics.ActionCopyObject, BucketOwner: parsedAcl.Owner, ObjectETag: etag, VersionId: res.VersionId, @@ -758,7 +723,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -768,7 +732,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -779,7 +742,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, }, }, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -789,7 +751,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -799,7 +760,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { if err != nil { return &Response{ MetaOpts: &MetaOptions{ - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, }, }, err @@ -850,7 +810,6 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { }, MetaOpts: &MetaOptions{ ContentLength: contentLength, - Action: metrics.ActionPutObject, BucketOwner: parsedAcl.Owner, ObjectETag: &res.ETag, ObjectSize: contentLength, diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go deleted file mode 100644 index 3e6d4345..00000000 --- a/s3api/controllers/utilities.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package controllers - -import ( - "encoding/xml" - "fmt" - "net/http" - - "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/metrics" - "github.com/versity/versitygw/s3api/debuglogger" - "github.com/versity/versitygw/s3api/utils" - "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3event" - "github.com/versity/versitygw/s3log" -) - -var ( - xmlhdr = []byte(`` + "\n") -) - -const ( - maxXMLBodyLen = 4 * 1024 * 1024 -) - -type MetaOptions struct { - ContentLength int64 - Action string - BucketOwner string - ObjectSize int64 - ObjectCount int64 - EventName s3event.EventType - ObjectETag *string - VersionId *string - Status int -} - -type Response struct { - Data any - Headers map[string]*string - MetaOpts *MetaOptions -} - -type Handler func(ctx *fiber.Ctx) (*Response, error) - -func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event.S3EventSender, mm *metrics.Manager) fiber.Handler { - return func(ctx *fiber.Ctx) error { - // if skip locals is set, skip to the next rout handler - if utils.ContextKeySkip.IsSet(ctx) { - utils.ContextKeySkip.Delete(ctx) - return ctx.Next() - } - - response, err := handler(ctx) - - // Set the response headers - SetResponseHeaders(ctx, response.Headers) - - opts := response.MetaOpts - // Send the metrics - if mm != nil { - if opts.ObjectCount > 0 { - mm.Send(ctx, err, opts.Action, opts.ObjectCount, opts.Status) - } else { - mm.Send(ctx, err, opts.Action, opts.ContentLength, opts.Status) - } - } - // Handle the error case - if err != nil { - // Audit the error log - if s3logger != nil { - s3logger.Log(ctx, err, nil, s3log.LogMeta{ - Action: opts.Action, - BucketOwner: opts.BucketOwner, - ObjectSize: opts.ObjectSize, - }) - } - serr, ok := err.(s3err.APIError) - if ok { - ctx.Status(serr.HTTPStatusCode) - return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", "")) - } - - debuglogger.Logf("Internal Error, %v", err) - ctx.Status(http.StatusInternalServerError) - - // If the error is not 's3err.APIError' return 'InternalError' - return ctx.Send(s3err.GetAPIErrorResponse( - s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) - } - - if opts.Status == 0 { - opts.Status = http.StatusOK - } - - // if no data payload is provided, send the response status - if response.Data == nil { - ctx.Status(opts.Status) - return nil - } - - var responseBytes []byte - - // Handle already encoded responses(text, json...) - encodedResp, ok := response.Data.([]byte) - if ok { - responseBytes = encodedResp - } else { - if responseBytes, err = xml.Marshal(response.Data); err != nil { - return err - } - - if len(responseBytes) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) - ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) - } - } - - if s3logger != nil { - s3logger.Log(ctx, nil, responseBytes, s3log.LogMeta{ - Action: opts.Action, - BucketOwner: opts.BucketOwner, - ObjectSize: opts.ObjectSize, - }) - } - - if s3evnt != nil { - s3evnt.SendEvent(ctx, s3event.EventMeta{ - BucketOwner: opts.BucketOwner, - ObjectSize: opts.ObjectSize, - ObjectETag: opts.ObjectETag, - VersionId: opts.VersionId, - EventName: opts.EventName, - }) - } - - if ok { - if len(responseBytes) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) - } - - return ctx.Send(responseBytes) - } - - msglen := len(xmlhdr) + len(responseBytes) - if msglen > maxXMLBodyLen { - debuglogger.Logf("XML encoded body len %v exceeds max len %v", - msglen, maxXMLBodyLen) - ctx.Status(http.StatusInternalServerError) - - return ctx.Send(s3err.GetAPIErrorResponse( - s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) - } - res := make([]byte, 0, msglen) - res = append(res, xmlhdr...) - res = append(res, responseBytes...) - - return ctx.Send(res) - } -} - -// Sets the response headers -func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { - if headers == nil { - return - } - for key, val := range headers { - if val == nil || *val == "" { - continue - } - ctx.Response().Header.Add(key, *val) - } -} - -// Returns MethodNotAllowed for unmatched routes -func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { - return &Response{ - MetaOpts: &MetaOptions{ - Action: metrics.ActionUndetected, - }, - }, s3err.GetAPIError(s3err.ErrMethodNotAllowed) -} diff --git a/s3api/middlewares/acl-parser.go b/s3api/middlewares/acl-parser.go index 8a8dceed..53f3ec8a 100644 --- a/s3api/middlewares/acl-parser.go +++ b/s3api/middlewares/acl-parser.go @@ -15,65 +15,26 @@ package middlewares import ( - "net/http" - "regexp" - "strings" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" - "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -var ( - singlePath = regexp.MustCompile(`^/[^/]+/?$`) -) - -func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler { +// ParseAcl retreives the bucket acl and stores in the context locals +// if no bucket is found, it returns 'NoSuchBucket' +func ParseAcl(be backend.Backend) fiber.Handler { return func(ctx *fiber.Ctx) error { - path := ctx.Path() - pathParts := strings.Split(path, "/") - bucket := pathParts[1] - if path == "/" { - return ctx.Next() - } - if ctx.Method() == http.MethodPatch { - return ctx.Next() - } - if singlePath.MatchString(path) && - ctx.Method() == http.MethodPut && - !ctx.Request().URI().QueryArgs().Has("acl") && - !ctx.Request().URI().QueryArgs().Has("tagging") && - !ctx.Request().URI().QueryArgs().Has("versioning") && - !ctx.Request().URI().QueryArgs().Has("policy") && - !ctx.Request().URI().QueryArgs().Has("object-lock") && - !ctx.Request().URI().QueryArgs().Has("ownershipControls") && - !ctx.Request().URI().QueryArgs().Has("cors") { - isRoot, acct := utils.ContextKeyIsRoot.Get(ctx).(bool), utils.ContextKeyAccount.Get(ctx).(auth.Account) - if err := auth.MayCreateBucket(acct, isRoot); err != nil { - return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"}) - } - if readonly { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAccessDenied), - &controllers.MetaOpts{ - Logger: logger, - Action: "CreateBucket", - }) - } - return ctx.Next() - } + bucket := ctx.Params("bucket") data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { - return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger}) + return err } parsedAcl, err := auth.ParseACL(data) if err != nil { - return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger}) + return err } // if owner is not set, set default owner to root account @@ -82,6 +43,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe } utils.ContextKeyParsedAcl.Set(ctx, parsedAcl) - return ctx.Next() + return nil } } diff --git a/s3api/middlewares/admin.go b/s3api/middlewares/admin.go index e5646cd0..b8626dff 100644 --- a/s3api/middlewares/admin.go +++ b/s3api/middlewares/admin.go @@ -15,46 +15,20 @@ package middlewares import ( - "strings" - "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" - "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -func IsAdmin(logger s3log.AuditLogger) fiber.Handler { +// IsAdmin is a middleware that restricts access to admin APIs, allowing only admin users +func IsAdmin(action string) fiber.Handler { return func(ctx *fiber.Ctx) error { acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) if acct.Role != auth.RoleAdmin { - path := ctx.Path() - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied), - &controllers.MetaOpts{ - Logger: logger, - Action: detectAction(path), - }) + return s3err.GetAPIError(s3err.ErrAdminAccessDenied) } - return ctx.Next() + return nil } } - -func detectAction(path string) (action string) { - if strings.Contains(path, "create-user") { - action = metrics.ActionAdminCreateUser - } else if strings.Contains(path, "update-user") { - action = metrics.ActionAdminUpdateUser - } else if strings.Contains(path, "delete-user") { - action = metrics.ActionAdminDeleteUser - } else if strings.Contains(path, "list-user") { - action = metrics.ActionAdminListUsers - } else if strings.Contains(path, "list-buckets") { - action = metrics.ActionAdminListBuckets - } else if strings.Contains(path, "change-bucket-owner") { - action = metrics.ActionAdminChangeBucketOwner - } - return action -} diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 54aa70a0..b9afa33f 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -25,11 +25,8 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" - "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) const ( @@ -42,45 +39,45 @@ type RootUserConfig struct { Secret string } -func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler { +func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler { acct := accounts{root: root, iam: iam} return func(ctx *fiber.Ctx) error { // The bucket is public, no need to check this signature if utils.ContextKeyPublicBucket.IsSet(ctx) { - return ctx.Next() + return nil } // If ContextKeyAuthenticated is set in context locals, it means it was presigned url case if utils.ContextKeyAuthenticated.IsSet(ctx) { - return ctx.Next() + return nil } authorization := ctx.Get("Authorization") if authorization == "" { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm) + return s3err.GetAPIError(s3err.ErrAuthHeaderEmpty) } authData, err := utils.ParseAuthorization(authorization) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } if authData.Region != region { - return sendResponse(ctx, s3err.APIError{ + return s3err.APIError{ Code: "SignatureDoesNotMatch", Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region), HTTPStatusCode: http.StatusForbidden, - }, logger, mm) + } } utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) account, err := acct.getAccount(authData.Access) if err == auth.ErrNoSuchUser { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm) + return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID) } if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } utils.ContextKeyAccount.Set(ctx, account) @@ -88,23 +85,23 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au // Check X-Amz-Date header date := ctx.Get("X-Amz-Date") if date == "" { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm) + return s3err.GetAPIError(s3err.ErrMissingDateHeader) } // Parse the date and check the date validity tdate, err := time.Parse(iso8601Format, date) if err != nil { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm) + return s3err.GetAPIError(s3err.ErrMalformedDate) } if date[:8] != authData.Date { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm) + return s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch) } // Validate the dates difference err = utils.ValidateDate(tdate) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } var contentLength int64 @@ -113,7 +110,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) //TODO: not sure if InvalidRequest should be returned in this case if err != nil { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm) + return s3err.GetAPIError(s3err.ErrInvalidRequest) } } @@ -136,23 +133,23 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au return cr }) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } - return ctx.Next() + return nil } // Content-Length has to be set for data uploads: PutObject, UploadPart if contentLengthStr == "" { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm) + return s3err.GetAPIError(s3err.ErrMissingContentLength) } // the upload limit for big data actions: PutObject, UploadPart // is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err if contentLength > maxObjSizeLimit { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm) + return s3err.GetAPIError(s3err.ErrEntityTooLarge) } - return ctx.Next() + return nil } if !utils.IsSpecialPayload(hashPayload) { @@ -162,16 +159,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au // Compare the calculated hash with the hash provided if hashPayload != hexPayload { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm) + return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch) } } err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } - return ctx.Next() + return nil } } @@ -185,13 +182,9 @@ func (a accounts) getAccount(access string) (auth.Account, error) { return auth.Account{ Access: a.root.Access, Secret: a.root.Secret, - Role: "admin", + Role: auth.RoleAdmin, }, nil } return a.iam.GetUserAccount(access) } - -func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error { - return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm}) -} diff --git a/s3api/middlewares/md5.go b/s3api/middlewares/md5.go index e1b81c65..5cd70b74 100644 --- a/s3api/middlewares/md5.go +++ b/s3api/middlewares/md5.go @@ -19,17 +19,15 @@ import ( "io" "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { +func VerifyMD5Body() fiber.Handler { return func(ctx *fiber.Ctx) error { incomingSum := ctx.Get("Content-Md5") if incomingSum == "" { - return ctx.Next() + return nil } if utils.IsBigDataAction(ctx) { @@ -39,18 +37,18 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler { return r }) if err != nil { - return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger}) + return err } - return ctx.Next() + return nil } sum := md5.Sum(ctx.Body()) calculatedSum := utils.Base64SumString(sum[:]) if incomingSum != calculatedSum { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger}) + return s3err.GetAPIError(s3err.ErrInvalidDigest) } - return ctx.Next() + return nil } } diff --git a/s3api/middlewares/presign-auth.go b/s3api/middlewares/presign-auth.go index f9d285cd..e872c5d8 100644 --- a/s3api/middlewares/presign-auth.go +++ b/s3api/middlewares/presign-auth.go @@ -20,22 +20,20 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler { +func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region string, debug bool) fiber.Handler { acct := accounts{root: root, iam: iam} return func(ctx *fiber.Ctx) error { // The bucket is public, no need to check this signature if utils.ContextKeyPublicBucket.IsSet(ctx) { - return ctx.Next() + return nil } if ctx.Query("X-Amz-Signature") == "" { - return ctx.Next() + return nil } // Set in the context the "authenticated" key, in case the authentication succeeds, @@ -44,17 +42,17 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger authData, err := utils.ParsePresignedURIParts(ctx) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) account, err := acct.getAccount(authData.Access) if err == auth.ErrNoSuchUser { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm) + return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID) } if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } utils.ContextKeyAccount.Set(ctx, account) @@ -64,32 +62,32 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) //TODO: not sure if InvalidRequest should be returned in this case if err != nil { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm) + return err } } if utils.IsBigDataAction(ctx) { // Content-Length has to be set for data uploads: PutObject, UploadPart if contentLengthStr == "" { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingContentLength), logger, mm) + return s3err.GetAPIError(s3err.ErrMissingContentLength) } // the upload limit for big data actions: PutObject, UploadPart // is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err if contentLength > maxObjSizeLimit { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrEntityTooLarge), logger, mm) + return s3err.GetAPIError(s3err.ErrEntityTooLarge) } wrapBodyReader(ctx, func(r io.Reader) io.Reader { return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug) }) - return ctx.Next() + return nil } err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug) if err != nil { - return sendResponse(ctx, err, logger, mm) + return err } - return ctx.Next() + return nil } } diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index e32e7557..30c7aae9 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -84,7 +84,7 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm if utils.IsUnsignedStreamingPayload(payloadType) { checksumType, err := utils.ExtractChecksumType(ctx) if err != nil { - return sendResponse(ctx, err, l, mm) + return err } wrapBodyReader(ctx, func(r io.Reader) io.Reader { @@ -93,7 +93,7 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm return cr }) if err != nil { - return sendResponse(ctx, err, l, mm) + return err } } } diff --git a/s3api/middlewares/url-decoder.go b/s3api/middlewares/url-decoder.go index fc275ac1..7c4b209b 100644 --- a/s3api/middlewares/url-decoder.go +++ b/s3api/middlewares/url-decoder.go @@ -18,19 +18,15 @@ import ( "net/url" "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/metrics" - "github.com/versity/versitygw/s3api/controllers" - "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) -func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { - return func(ctx *fiber.Ctx) error { - unescp, err := url.PathUnescape(string(ctx.Request().URI().PathOriginal())) - if err != nil { - return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm}) - } - ctx.Path(unescp) - return ctx.Next() +// DecodeURL url path unescapes the request url for the gateway +// to handle some special characters +func DecodeURL(ctx *fiber.Ctx) error { + unescp, err := url.PathUnescape(string(ctx.Request().URI().PathOriginal())) + if err != nil { + return err } + ctx.Path(unescp) + return nil } diff --git a/s3api/router.go b/s3api/router.go index 047d35aa..567ab62d 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -29,105 +29,679 @@ type S3ApiRouter struct { WithAdmSrv bool } -func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) { +func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool, region string, root middlewares.RootUserConfig) { ctrl := controllers.New(be, iam, logger, evs, mm, debug, readonly) + adminServices := &controllers.Services{ + Logger: aLogger, + } if sa.WithAdmSrv { adminController := controllers.NewAdminController(iam, be, aLogger) // CreateUser admin api - app.Patch("/create-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.CreateUser, aLogger, nil, nil)) + app.Patch("/create-user", + controllers.ProcessHandlers(adminController.CreateUser, metrics.ActionAdminCreateUser, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminCreateUser), + )) // DeleteUsers admin api - app.Patch("/delete-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.DeleteUser, aLogger, nil, nil)) + app.Patch("/delete-user", + controllers.ProcessHandlers(adminController.DeleteUser, metrics.ActionAdminDeleteUser, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminDeleteUser), + )) // UpdateUser admin api - app.Patch("/update-user", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.UpdateUser, aLogger, nil, nil)) + app.Patch("/update-user", + controllers.ProcessHandlers(adminController.UpdateUser, metrics.ActionAdminUpdateUser, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminUpdateUser), + )) // ListUsers admin api - app.Patch("/list-users", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ListUsers, aLogger, nil, nil)) + app.Patch("/list-users", + controllers.ProcessHandlers(adminController.ListUsers, metrics.ActionAdminListUsers, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminListUsers), + )) // ChangeBucketOwner admin api - app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ChangeBucketOwner, aLogger, nil, nil)) + app.Patch("/change-bucket-owner", + controllers.ProcessHandlers(adminController.ChangeBucketOwner, metrics.ActionAdminChangeBucketOwner, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminChangeBucketOwner), + )) // ListBucketsAndOwners admin api - app.Patch("/list-buckets", middlewares.IsAdmin(logger), controllers.ProcessResponse(adminController.ListBuckets, aLogger, nil, nil)) + app.Patch("/list-buckets", + controllers.ProcessHandlers(adminController.ListBuckets, metrics.ActionAdminListBuckets, adminServices, + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.IsAdmin(metrics.ActionAdminListBuckets), + )) + } + + services := &controllers.Services{ + Logger: logger, + EventSender: evs, + MetricsManager: mm, } // ListBuckets action - app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) + app.Get("/", + controllers.ProcessHandlers( + ctrl.ListBuckets, + metrics.ActionListAllMyBuckets, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListAllMyBuckets, "", auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + )) bucketRouter := app.Group("/:bucket") objectRouter := app.Group("/:bucket/*") // PUT bucket operations - bucketRouter.Put("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.PutObjectLockConfiguration, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.PutBucketCors, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.PutBucketPolicy, logger, evs, mm)) - bucketRouter.Put("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutBucketAcl, logger, evs, mm)) - bucketRouter.Put("", controllers.ProcessResponse(ctrl.CreateBucket, logger, evs, mm)) + bucketRouter.Put("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.PutBucketTagging, + metrics.ActionPutBucketTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("ownershipControls"), + controllers.ProcessHandlers( + ctrl.PutBucketOwnershipControls, + metrics.ActionPutBucketOwnershipControls, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketOwnershipControls, auth.PutBucketOwnershipControlsAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("versioning"), + controllers.ProcessHandlers( + ctrl.PutBucketVersioning, + metrics.ActionPutBucketVersioning, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketVersioning, auth.PutBucketVersioningAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("object-lock"), + controllers.ProcessHandlers( + ctrl.PutObjectLockConfiguration, + metrics.ActionPutObjectLockConfiguration, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectLockConfiguration, auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("cors"), + controllers.ProcessHandlers( + ctrl.PutBucketCors, + metrics.ActionPutBucketCors, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketCors, auth.PutBucketCorsAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("policy"), + controllers.ProcessHandlers( + ctrl.PutBucketPolicy, + metrics.ActionPutBucketPolicy, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketPolicy, auth.PutBucketPolicyAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + middlewares.MatchQueryArgs("acl"), + controllers.ProcessHandlers( + ctrl.PutBucketAcl, + metrics.ActionPutBucketAcl, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketAcl, auth.PutBucketAclAction, auth.PermissionWriteAcp), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Put("", + controllers.ProcessHandlers( + ctrl.CreateBucket, + metrics.ActionCreateBucket, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCreateBucket, auth.CreateBucketAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + )) // HeadBucket action - bucketRouter.Head("", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) + bucketRouter.Head("", + controllers.ProcessHandlers( + ctrl.HeadBucket, + metrics.ActionHeadBucket, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionHeadBucket, auth.ListBucketAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // DELETE bucket operations - bucketRouter.Delete("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteBucketTagging, logger, evs, mm)) - bucketRouter.Delete("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.DeleteBucketOwnershipControls, logger, evs, mm)) - bucketRouter.Delete("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.DeleteBucketPolicy, logger, evs, mm)) - bucketRouter.Delete("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm)) - bucketRouter.Delete("", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm)) + bucketRouter.Delete("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.DeleteBucketTagging, + metrics.ActionDeleteBucketTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Delete("", + middlewares.MatchQueryArgs("ownershipControls"), + controllers.ProcessHandlers( + ctrl.DeleteBucketOwnershipControls, + metrics.ActionDeleteBucketOwnershipControls, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketOwnershipControls, auth.PutBucketOwnershipControlsAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Delete("", + middlewares.MatchQueryArgs("policy"), + controllers.ProcessHandlers( + ctrl.DeleteBucketPolicy, + metrics.ActionDeleteBucketPolicy, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketPolicy, auth.PutBucketPolicyAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Delete("", + middlewares.MatchQueryArgs("cors"), + controllers.ProcessHandlers( + ctrl.DeleteBucketCors, + metrics.ActionDeleteBucketCors, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketCors, auth.PutBucketCorsAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Delete("", + controllers.ProcessHandlers( + ctrl.DeleteBucket, + metrics.ActionDeleteBucket, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucket, auth.DeleteBucketAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // GET bucket operations - bucketRouter.Get("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.GetBucketPolicy, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.GetBucketCors, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.GetObjectLockConfiguration, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetBucketAcl, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.ListMultipartUploads, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgs("versions"), controllers.ProcessResponse(ctrl.ListObjectVersions, logger, evs, mm)) - bucketRouter.Get("", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) - bucketRouter.Get("", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) + bucketRouter.Get("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.GetBucketTagging, + metrics.ActionGetBucketTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketTagging, auth.GetBucketTaggingAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("ownershipControls"), + controllers.ProcessHandlers( + ctrl.GetBucketOwnershipControls, + metrics.ActionGetBucketOwnershipControls, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketOwnershipControls, auth.GetBucketOwnershipControlsAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("versioning"), + controllers.ProcessHandlers( + ctrl.GetBucketVersioning, + metrics.ActionGetBucketVersioning, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketVersioning, auth.GetBucketVersioningAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("policy"), + controllers.ProcessHandlers( + ctrl.GetBucketPolicy, + metrics.ActionGetBucketPolicy, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketPolicy, auth.GetBucketPolicyAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("cors"), + controllers.ProcessHandlers( + ctrl.GetBucketCors, + metrics.ActionGetBucketCors, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketCors, auth.GetBucketCorsAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("object-lock"), + controllers.ProcessHandlers( + ctrl.GetObjectLockConfiguration, + metrics.ActionGetObjectLockConfiguration, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectLockConfiguration, auth.GetBucketObjectLockConfigurationAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("acl"), + controllers.ProcessHandlers( + ctrl.GetBucketAcl, + metrics.ActionGetBucketAcl, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketAcl, auth.GetBucketAclAction, auth.PermissionReadAcp), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("uploads"), + controllers.ProcessHandlers( + ctrl.ListMultipartUploads, + metrics.ActionListMultipartUploads, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListMultipartUploads, auth.ListBucketMultipartUploadsAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgs("versions"), + controllers.ProcessHandlers( + ctrl.ListObjectVersions, + metrics.ActionListObjectVersions, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjectVersions, auth.ListBucketVersionsAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + middlewares.MatchQueryArgWithValue("list-type", "2"), + controllers.ProcessHandlers( + ctrl.ListObjectsV2, + metrics.ActionListObjectsV2, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjectsV2, auth.ListBucketAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + bucketRouter.Get("", + controllers.ProcessHandlers( + ctrl.ListObjects, + metrics.ActionListObjects, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjects, auth.ListBucketAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // DeleteObjects action - bucketRouter.Post("", middlewares.MatchQueryArgs("delete"), controllers.ProcessResponse(ctrl.DeleteObjects, logger, evs, mm)) + bucketRouter.Post("", + middlewares.MatchQueryArgs("delete"), + controllers.ProcessHandlers( + ctrl.DeleteObjects, + metrics.ActionDeleteObjects, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObjects, auth.DeleteObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // HeadObject - objectRouter.Head("", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) + objectRouter.Head("", + controllers.ProcessHandlers( + ctrl.HeadObject, + metrics.ActionHeadObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionHeadObject, auth.GetObjectAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // GET object operations - objectRouter.Get("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetObjectTagging, logger, evs, mm)) - objectRouter.Get("", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.GetObjectRetention, logger, evs, mm)) - objectRouter.Get("", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.GetObjectLegalHold, logger, evs, mm)) - objectRouter.Get("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetObjectAcl, logger, evs, mm)) - objectRouter.Get("", middlewares.MatchQueryArgs("attributes"), controllers.ProcessResponse(ctrl.GetObjectAttributes, logger, evs, mm)) - objectRouter.Get("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.ListParts, logger, evs, mm)) - objectRouter.Get("", controllers.ProcessResponse(ctrl.GetObject, logger, evs, mm)) + objectRouter.Get("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.GetObjectTagging, + metrics.ActionGetObjectTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectTagging, auth.GetObjectTaggingAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + middlewares.MatchQueryArgs("retention"), + controllers.ProcessHandlers( + ctrl.GetObjectRetention, + metrics.ActionGetObjectRetention, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectRetention, auth.GetObjectRetentionAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + middlewares.MatchQueryArgs("legal-hold"), + controllers.ProcessHandlers( + ctrl.GetObjectLegalHold, + metrics.ActionGetObjectLegalHold, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectLegalHold, auth.GetObjectLegalHoldAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + middlewares.MatchQueryArgs("acl"), + controllers.ProcessHandlers( + ctrl.GetObjectAcl, + metrics.ActionGetObjectAcl, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectAcl, auth.GetObjectAclAction, auth.PermissionReadAcp), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + middlewares.MatchQueryArgs("attributes"), + controllers.ProcessHandlers( + ctrl.GetObjectAttributes, + metrics.ActionGetObjectAttributes, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectAttributes, auth.GetObjectAttributesAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + middlewares.MatchQueryArgs("uploadId"), + controllers.ProcessHandlers( + ctrl.ListParts, + metrics.ActionListParts, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListParts, auth.ListMultipartUploadPartsAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Get("", + controllers.ProcessHandlers( + ctrl.GetObject, + metrics.ActionGetObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObject, auth.GetObjectAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // DELETE object operations - objectRouter.Delete("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.DeleteObjectTagging, logger, evs, mm)) - objectRouter.Delete("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.AbortMultipartUplaod, logger, evs, mm)) - objectRouter.Delete("", controllers.ProcessResponse(ctrl.DeleteObject, logger, evs, mm)) + objectRouter.Delete("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.DeleteObjectTagging, + metrics.ActionDeleteObjectTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObjectTagging, auth.DeleteObjectTaggingAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Delete("", + middlewares.MatchQueryArgs("uploadId"), + controllers.ProcessHandlers( + ctrl.AbortMultipartUpload, + metrics.ActionAbortMultipartUpload, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionAbortMultipartUpload, auth.AbortMultipartUploadAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Delete("", + controllers.ProcessHandlers( + ctrl.DeleteObject, + metrics.ActionDeleteObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObject, auth.DeleteObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) - objectRouter.Post("", middlewares.MatchQueryArgs("restore"), controllers.ProcessResponse(ctrl.RestoreObject, logger, evs, mm)) - objectRouter.Post("", middlewares.MatchQueryArgs("list-type"), middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.RestoreObject, logger, evs, mm)) - objectRouter.Post("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.CompleteMultipartUpload, logger, evs, mm)) - objectRouter.Post("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.CreateMultipartUpload, logger, evs, mm)) + objectRouter.Post("", + middlewares.MatchQueryArgs("restore"), + controllers.ProcessHandlers( + ctrl.RestoreObject, + metrics.ActionRestoreObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionRestoreObject, auth.RestoreObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Post("", + middlewares.MatchQueryArgs("select"), + middlewares.MatchQueryArgWithValue("select-type", "2"), + controllers.ProcessHandlers( + ctrl.SelectObjectContent, + metrics.ActionSelectObjectContent, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionSelectObjectContent, auth.GetObjectAction, auth.PermissionRead), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Post("", + middlewares.MatchQueryArgs("uploadId"), + controllers.ProcessHandlers( + ctrl.CompleteMultipartUpload, + metrics.ActionCompleteMultipartUpload, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCompleteMultipartUpload, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Post("", + middlewares.MatchQueryArgs("uploads"), + controllers.ProcessHandlers( + ctrl.CreateMultipartUpload, + metrics.ActionCreateMultipartUpload, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCreateMultipartUpload, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // PUT object operations - objectRouter.Put("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutObjectTagging, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.PutObjectRetention, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.PutObjectLegalHold, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutObjectAcl, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.UploadPartCopy, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), controllers.ProcessResponse(ctrl.UploadPart, logger, evs, mm)) - objectRouter.Put("", middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.CopyObject, logger, evs, mm)) - objectRouter.Put("", controllers.ProcessResponse(ctrl.PutObject, logger, evs, mm)) + objectRouter.Put("", + middlewares.MatchQueryArgs("tagging"), + controllers.ProcessHandlers( + ctrl.PutObjectTagging, + metrics.ActionPutObjectTagging, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectTagging, auth.PutObjectTaggingAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchQueryArgs("retention"), + controllers.ProcessHandlers( + ctrl.PutObjectRetention, + metrics.ActionPutObjectRetention, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectRetention, auth.PutObjectRetentionAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchQueryArgs("legal-hold"), + controllers.ProcessHandlers( + ctrl.PutObjectLegalHold, + metrics.ActionPutObjectLegalHold, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectLegalHold, auth.PutObjectLegalHoldAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchQueryArgs("acl"), + controllers.ProcessHandlers( + ctrl.PutObjectAcl, + metrics.ActionPutObjectAcl, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectAcl, auth.PutObjectAclAction, auth.PermissionWriteAcp), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchQueryArgs("uploadId", "partNumber"), + middlewares.MatchHeader("X-Amz-Copy-Source"), + controllers.ProcessHandlers( + ctrl.UploadPartCopy, + metrics.ActionUploadPartCopy, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionUploadPartCopy, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchQueryArgs("uploadId", "partNumber"), + controllers.ProcessHandlers( + ctrl.UploadPart, + metrics.ActionUploadPart, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionUploadPart, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + middlewares.MatchHeader("X-Amz-Copy-Source"), + controllers.ProcessHandlers( + ctrl.CopyObject, + metrics.ActionCopyObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCopyObject, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) + objectRouter.Put("", + controllers.ProcessHandlers( + ctrl.PutObject, + metrics.ActionPutObject, + services, + middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObject, auth.PutObjectAction, auth.PermissionWrite), + middlewares.VerifyPresignedV4Signature(root, iam, region, debug), + middlewares.VerifyV4Signature(root, iam, region, debug), + middlewares.VerifyMD5Body(), + middlewares.ParseAcl(be), + )) // Return MethodNotAllowed for all the unmatched routes - app.All("*", controllers.ProcessResponse(ctrl.HandleUnmatch, logger, evs, mm)) + app.All("*", controllers.ProcessHandlers(ctrl.HandleUnmatch, metrics.ActionUndetected, services)) } diff --git a/s3api/router_test.go b/s3api/router_test.go index 58f01b31..d4ca3351 100644 --- a/s3api/router_test.go +++ b/s3api/router_test.go @@ -20,6 +20,7 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/s3api/middlewares" ) func TestS3ApiRouter_Init(t *testing.T) { @@ -45,7 +46,7 @@ func TestS3ApiRouter_Init(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false) + tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false, "us-east-1", middlewares.RootUserConfig{}) }) } } diff --git a/s3api/server.go b/s3api/server.go index 4583e9da..26c2639d 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -23,6 +23,7 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/middlewares" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" @@ -76,7 +77,10 @@ func New( return ctx.SendStatus(http.StatusOK) }) } - app.Use(middlewares.DecodeURL(l, mm)) + + // initialize the 'DecodeURL' middleware which + // path unescapes the url + app.Use(controllers.WrapMiddleware(middlewares.DecodeURL, l, mm)) // initialize host-style parser in virtual domain is specified if server.virtualDomain != "" { diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index 19b8d269..9b965b77 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -130,6 +130,10 @@ func (c *S3Conf) GetClient() *s3.Client { }) } +func (c *S3Conf) GetPresignClient() *s3.PresignClient { + return s3.NewPresignClient(c.GetClient()) +} + func (c *S3Conf) GetAnonymousClient() *s3.Client { cfg := c.Config() cfg.Credentials = aws.AnonymousCredentials{} diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 74345044..d14b40e6 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -51,7 +51,6 @@ func Authentication_invalid_auth_header(s *S3Conf) error { testName := "Authentication_invalid_auth_header" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -76,7 +75,6 @@ func Authentication_unsupported_signature_version(s *S3Conf) error { testName := "Authentication_unsupported_signature_version" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -103,7 +101,6 @@ func Authentication_malformed_credentials(s *S3Conf) error { testName := "Authentication_malformed_credentials" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -131,7 +128,6 @@ func Authentication_malformed_credentials_invalid_parts(s *S3Conf) error { testName := "Authentication_malformed_credentials_invalid_parts" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -159,7 +155,6 @@ func Authentication_credentials_terminated_string(s *S3Conf) error { testName := "Authentication_credentials_terminated_string" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -187,7 +182,6 @@ func Authentication_credentials_incorrect_service(s *S3Conf) error { testName := "Authentication_credentials_incorrect_service" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "ec2", @@ -217,7 +211,6 @@ func Authentication_credentials_incorrect_region(s *S3Conf) error { } return authHandler(&cfg, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -246,7 +239,6 @@ func Authentication_credentials_invalid_date(s *S3Conf) error { testName := "Authentication_credentials_invalid_date" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -274,7 +266,6 @@ func Authentication_credentials_future_date(s *S3Conf) error { testName := "Authentication_credentials_future_date" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -312,7 +303,6 @@ func Authentication_credentials_past_date(s *S3Conf) error { testName := "Authentication_credentials_past_date" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -350,7 +340,6 @@ func Authentication_credentials_non_existing_access_key(s *S3Conf) error { testName := "Authentication_credentials_non_existing_access_key" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -378,7 +367,6 @@ func Authentication_invalid_signed_headers(s *S3Conf) error { testName := "Authentication_invalid_signed_headers" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -406,7 +394,6 @@ func Authentication_missing_date_header(s *S3Conf) error { testName := "Authentication_missing_date_header" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -431,7 +418,6 @@ func Authentication_invalid_date_header(s *S3Conf) error { testName := "Authentication_invalid_date_header" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -456,7 +442,6 @@ func Authentication_date_mismatch(s *S3Conf) error { testName := "Authentication_date_mismatch" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -481,7 +466,6 @@ func Authentication_incorrect_payload_hash(s *S3Conf) error { testName := "Authentication_incorrect_payload_hash" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -506,7 +490,6 @@ func Authentication_incorrect_md5(s *S3Conf) error { testName := "Authentication_incorrect_md5" return authHandler(s, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -534,7 +517,6 @@ func Authentication_signature_error_incorrect_secret_key(s *S3Conf) error { cfg.awsSecret = s.awsSecret + "a" return authHandler(&cfg, &authConfig{ testName: testName, - path: "my-bucket", method: http.MethodGet, body: nil, service: "s3", @@ -556,9 +538,9 @@ func Authentication_signature_error_incorrect_secret_key(s *S3Conf) error { func PresignedAuth_unsupported_algorithm(s *S3Conf) error { testName := "PresignedAuth_unsupported_algorithm" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -586,9 +568,9 @@ func PresignedAuth_unsupported_algorithm(s *S3Conf) error { func PresignedAuth_missing_credentials_query_param(s *S3Conf) error { testName := "PresignedAuth_missing_credentials_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -623,9 +605,9 @@ func PresignedAuth_missing_credentials_query_param(s *S3Conf) error { func PresignedAuth_malformed_creds_invalid_parts(s *S3Conf) error { testName := "PresignedAuth_malformed_creds_invalid_parts" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -660,9 +642,9 @@ func PresignedAuth_malformed_creds_invalid_parts(s *S3Conf) error { func PresignedAuth_creds_invalid_terminator(s *S3Conf) error { testName := "PresignedAuth_creds_invalid_terminator" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -693,9 +675,9 @@ func PresignedAuth_creds_invalid_terminator(s *S3Conf) error { func PresignedAuth_creds_incorrect_service(s *S3Conf) error { testName := "PresignedAuth_creds_incorrect_service" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -726,17 +708,19 @@ func PresignedAuth_creds_incorrect_service(s *S3Conf) error { func PresignedAuth_creds_incorrect_region(s *S3Conf) error { testName := "PresignedAuth_creds_incorrect_region" - cfg := *s + return presignedAuthHandler(s, testName, func(_ *s3.PresignClient, bucket string) error { + cfg := *s - if cfg.awsRegion == "us-east-1" { - cfg.awsRegion = "us-west-1" - } else { - cfg.awsRegion = "us-east-1" - } + if cfg.awsRegion == "us-east-1" { + cfg.awsRegion = "us-west-1" + } else { + cfg.awsRegion = "us-east-1" + } + + client := cfg.GetPresignClient() - return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -766,9 +750,9 @@ func PresignedAuth_creds_incorrect_region(s *S3Conf) error { func PresignedAuth_creds_invalid_date(s *S3Conf) error { testName := "PresignedAuth_creds_invalid_date" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -799,9 +783,9 @@ func PresignedAuth_creds_invalid_date(s *S3Conf) error { func PresignedAuth_non_existing_access_key_id(s *S3Conf) error { testName := "PresignedAuth_non_existing_access_key_id" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -832,9 +816,9 @@ func PresignedAuth_non_existing_access_key_id(s *S3Conf) error { func PresignedAuth_missing_date_query(s *S3Conf) error { testName := "PresignedAuth_missing_date_query" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -869,9 +853,9 @@ func PresignedAuth_missing_date_query(s *S3Conf) error { func PresignedAuth_dates_mismatch(s *S3Conf) error { testName := "PresignedAuth_dates_mismatch" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -902,9 +886,9 @@ func PresignedAuth_dates_mismatch(s *S3Conf) error { func PresignedAuth_missing_signed_headers_query_param(s *S3Conf) error { testName := "PresignedAuth_missing_signed_headers_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -939,9 +923,9 @@ func PresignedAuth_missing_signed_headers_query_param(s *S3Conf) error { func PresignedAuth_missing_expiration_query_param(s *S3Conf) error { testName := "PresignedAuth_missing_expiration_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -976,9 +960,9 @@ func PresignedAuth_missing_expiration_query_param(s *S3Conf) error { func PresignedAuth_invalid_expiration_query_param(s *S3Conf) error { testName := "PresignedAuth_invalid_expiration_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -1013,9 +997,9 @@ func PresignedAuth_invalid_expiration_query_param(s *S3Conf) error { func PresignedAuth_negative_expiration_query_param(s *S3Conf) error { testName := "PresignedAuth_negative_expiration_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -1050,9 +1034,9 @@ func PresignedAuth_negative_expiration_query_param(s *S3Conf) error { func PresignedAuth_exceeding_expiration_query_param(s *S3Conf) error { testName := "PresignedAuth_exceeding_expiration_query_param" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -1087,9 +1071,9 @@ func PresignedAuth_exceeding_expiration_query_param(s *S3Conf) error { func PresignedAuth_expired_request(s *S3Conf) error { testName := "PresignedAuth_expired_request" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -1131,11 +1115,13 @@ func PresignedAuth_expired_request(s *S3Conf) error { func PresignedAuth_incorrect_secret_key(s *S3Conf) error { testName := "PresignedAuth_incorrect_secret_key" - cfg := *s - cfg.awsSecret += "x" - return presignedAuthHandler(&cfg, testName, func(client *s3.PresignClient) error { + return presignedAuthHandler(s, testName, func(_ *s3.PresignClient, bucket string) error { + cfg := *s + cfg.awsSecret += "x" + client := cfg.GetPresignClient() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: getPtr("my-bucket")}) + v4req, err := client.PresignDeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &bucket}) cancel() if err != nil { return err @@ -1161,12 +1147,7 @@ func PresignedAuth_incorrect_secret_key(s *S3Conf) error { func PresignedAuth_PutObject_success(s *S3Conf) error { testName := "PresignedAuth_PutObject_success" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { - bucket := getBucketName() - err := setup(s, bucket) - if err != nil { - return err - } + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: getPtr("my-obj")}) cancel() @@ -1188,23 +1169,14 @@ func PresignedAuth_PutObject_success(s *S3Conf) error { return fmt.Errorf("expected my-obj to be successfully uploaded and get 200 response status, instead got %v", resp.StatusCode) } - err = teardown(s, bucket) - if err != nil { - return err - } - return nil }) } func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error { testName := "PresignedAuth_Put_GetObject_with_data" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { - bucket, obj := getBucketName(), "my-obj" - err := setup(s, bucket) - if err != nil { - return err - } + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { + obj := "my-obj" data := "Hello world" body := strings.NewReader(data) @@ -1263,23 +1235,14 @@ func PresignedAuth_Put_GetObject_with_data(s *S3Conf) error { return fmt.Errorf("expected get object response body to be %v, instead got %s", data, respBody) } - err = teardown(s, bucket) - if err != nil { - return err - } - return nil }) } func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error { testName := "PresignedAuth_Put_GetObject_with_UTF8_chars" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { - bucket, obj := getBucketName(), "my-$%^&*;" - err := setup(s, bucket) - if err != nil { - return err - } + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { + obj := "my-$%^&*;" ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj}) @@ -1325,19 +1288,14 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error { return fmt.Errorf("expected get object response status to be %v, instead got %v", http.StatusOK, resp.StatusCode) } - err = teardown(s, bucket) - if err != nil { - return err - } - return nil }) } func PresignedAuth_UploadPart(s *S3Conf) error { testName := "PresignedAuth_UploadPart" - return presignedAuthHandler(s, testName, func(client *s3.PresignClient) error { - bucket, key, partNumber := getBucketName(), "my-mp", int32(1) + return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { + key, partNumber := "my-mp", int32(1) err := setup(s, bucket) if err != nil { @@ -17445,7 +17403,7 @@ func PublicBucket_public_acl(s *S3Conf) error { }) return err }, - ExpectedErr: s3err.GetAPIError(s3err.ErrNotImplemented), + ExpectedErr: s3err.GetAPIError(s3err.ErrAccessDenied), }, { Action: "GetObjectAttributes", diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 569724db..8edcb7bc 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -284,11 +284,23 @@ func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) err return nil } -func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error { +func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient, bucket string) error) error { runF(testName) - clt := s3.NewPresignClient(s.GetClient()) + bucket := getBucketName() + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + clt := s.GetPresignClient() - err := handler(clt) + err = handler(clt, bucket) + if err != nil { + failF("%v: %v", testName, err) + return fmt.Errorf("%v: %w", testName, err) + } + + err = teardown(s, bucket) if err != nil { failF("%v: %v", testName, err) return fmt.Errorf("%v: %w", testName, err) From 65cd44aaddc116a5f233f3291b0c599a621eb871 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Tue, 1 Jul 2025 01:59:21 +0400 Subject: [PATCH 12/20] fix: fixes the s3 access logs and metrics manager reporting. Fixes the default cotext keys setter order in the middlewares. --- s3api/controllers/base.go | 21 ++++++++++++--------- s3api/server.go | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 15713d2c..1ccf4e5f 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -2004,16 +2004,16 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { return func(ctx *fiber.Ctx) error { err := handler(ctx) - if mm != nil { - mm.Send(ctx, err, metrics.ActionUndetected, 0, 0) - } - if logger != nil { - logger.Log(ctx, err, ctx.Body(), s3log.LogMeta{ - Action: metrics.ActionUndetected, - }) - } - if err != nil { + if mm != nil { + mm.Send(ctx, err, metrics.ActionUndetected, 0, 0) + } + if logger != nil { + logger.Log(ctx, err, ctx.Body(), s3log.LogMeta{ + Action: metrics.ActionUndetected, + }) + } + serr, ok := err.(s3err.APIError) if ok { ctx.Status(serr.HTTPStatusCode) @@ -2041,6 +2041,9 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s SetResponseHeaders(ctx, response.Headers) opts := response.MetaOpts + if opts == nil { + opts = &MetaOptions{} + } // Send the metrics if svc.MetricsManager != nil { if opts.ObjectCount > 0 { diff --git a/s3api/server.go b/s3api/server.go index 26c2639d..857bf810 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -78,6 +78,9 @@ func New( }) } + // initilaze the default value setter middleware + app.Use(middlewares.SetDefaultValues(root, region)) + // initialize the 'DecodeURL' middleware which // path unescapes the url app.Use(controllers.WrapMiddleware(middlewares.DecodeURL, l, mm)) @@ -87,9 +90,6 @@ func New( app.Use(middlewares.HostStyleParser(server.virtualDomain)) } - // initilaze the default value setter middleware - app.Use(middlewares.SetDefaultValues(root, region)) - // initialize the debug logger in debug mode if server.debug { app.Use(middlewares.DebugLogger()) From 866b07b98fce33473bdfd5791fd2d362213dca3b Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 7 Jul 2025 21:23:27 +0400 Subject: [PATCH 13/20] feat: implementes unit tests for all the bucket action controllers. --- auth/acl.go | 2 +- go.mod | 1 + go.sum | 11 +- s3api/controllers/base_test.go | 150 +++ s3api/controllers/bucket-delete_test.go | 411 +++++++++ s3api/controllers/bucket-get_test.go | 1120 +++++++++++++++++++++++ s3api/controllers/bucket-head_test.go | 136 +++ s3api/controllers/bucket-list.go | 1 - s3api/controllers/bucket-list_test.go | 112 +++ s3api/controllers/bucket-post.go | 25 +- s3api/controllers/bucket-post_test.go | 165 ++++ s3api/controllers/bucket-put_test.go | 813 ++++++++++++++++ tests/integration/tests.go | 10 - 13 files changed, 2931 insertions(+), 26 deletions(-) create mode 100644 s3api/controllers/base_test.go create mode 100644 s3api/controllers/bucket-delete_test.go create mode 100644 s3api/controllers/bucket-get_test.go create mode 100644 s3api/controllers/bucket-head_test.go create mode 100644 s3api/controllers/bucket-list_test.go create mode 100644 s3api/controllers/bucket-post_test.go create mode 100644 s3api/controllers/bucket-put_test.go diff --git a/auth/acl.go b/auth/acl.go index e9003243..79ac6e18 100644 --- a/auth/acl.go +++ b/auth/acl.go @@ -385,7 +385,7 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) { for _, acc := range accs { _, err := iam.GetUserAccount(acc) if err != nil { - if err == ErrNoSuchUser { + if err == ErrNoSuchUser || err == s3err.GetAPIError(s3err.ErrAdminUserNotFound) { result = append(result, acc) continue } diff --git a/go.mod b/go.mod index e70ce217..6dc68705 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect golang.org/x/crypto v0.40.0 // indirect golang.org/x/net v0.42.0 // indirect diff --git a/go.sum b/go.sum index ee78aeb7..2aa7aeea 100644 --- a/go.sum +++ b/go.sum @@ -118,6 +118,10 @@ github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXw github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -152,6 +156,8 @@ github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -163,8 +169,9 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -259,6 +266,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go new file mode 100644 index 00000000..4f0173d6 --- /dev/null +++ b/s3api/controllers/base_test.go @@ -0,0 +1,150 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "bytes" + "net/http" + "net/http/httptest" + "net/url" + "path" + "testing" + + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +var ( + defaultLocals map[utils.ContextKey]any = map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + } + + accessDeniedLocals map[utils.ContextKey]any = map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + } +) + +type testInput struct { + bucket string + object string + body []byte + locals map[utils.ContextKey]any + headers map[string]string + queries map[string]string + beRes any + beErr error + extraMockErr error +} + +type testOutput struct { + response *Response + err error +} + +type ctxInputs struct { + bucket string + object string + body []byte + locals map[utils.ContextKey]any + headers map[string]string + queries map[string]string +} + +func testController(t *testing.T, ctrl Controller, resp *Response, expectedErr error, input ctxInputs) { + app := fiber.New() + + app.Post("/:bucket/*", func(ctx *fiber.Ctx) error { + // set the request body + ctx.Request().SetBody(input.body) + // set the request locals + if input.locals != nil { + for key, local := range input.locals { + key.Set(ctx, local) + } + } + + // call the controller by passing the ctx + res, err := ctrl(ctx) + assert.Equal(t, resp, res) + if expectedErr != nil { + assert.Error(t, err) + + switch expectedErr.(type) { + case s3err.APIError: + assert.EqualValues(t, expectedErr, err) + default: + assert.ErrorContains(t, err, expectedErr.Error()) + } + } else { + assert.NoError(t, err) + } + + return nil + }) + + req := buildRequest(input.bucket, input.object, input.body, input.headers, input.queries) + + _, err := app.Test(req) + assert.NoError(t, err) +} + +func buildRequest(bucket, object string, body []byte, headers, queries map[string]string) *http.Request { + if bucket == "" { + bucket = "bucket" + } + if object == "" { + object = "object" + } + uri := url.URL{ + Path: "/" + path.Join(bucket, object), + } + + // set the request query params + if queries != nil { + q := uri.Query() + for key, val := range queries { + q.Set(key, val) + } + + uri.RawQuery = q.Encode() + } + + // create a new request + req := httptest.NewRequest(http.MethodPost, uri.String(), bytes.NewReader(body)) + + // set the request headers + for key, val := range headers { + req.Header.Add(key, val) + } + + return req +} diff --git a/s3api/controllers/bucket-delete_test.go b/s3api/controllers/bucket-delete_test.go new file mode 100644 index 00000000..19eb95da --- /dev/null +++ b/s3api/controllers/bucket-delete_test.go @@ -0,0 +1,411 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "net/http" + "testing" + + "github.com/versity/versitygw/s3err" +) + +func TestS3ApiController_DeleteBucketTagging(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrAclNotSupported), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrAclNotSupported), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteBucketTaggingFunc: func(_ context.Context, _ string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteBucketTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_DeleteBucketOwnershipControls(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteBucketOwnershipControls, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_DeleteBucketPolicy(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidDigest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidDigest), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteBucketPolicy, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_DeleteBucketCors(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrAdminMethodNotSupported), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAdminMethodNotSupported), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteBucketCorsFunc: func(contextMoqParam context.Context, bucket string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteBucketCors, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_DeleteBucket(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidDigest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidDigest), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteBucketFunc: func(contextMoqParam context.Context, bucket string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteBucket, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} diff --git a/s3api/controllers/bucket-get_test.go b/s3api/controllers/bucket-get_test.go new file mode 100644 index 00000000..1787863d --- /dev/null +++ b/s3api/controllers/bucket-get_test.go @@ -0,0 +1,1120 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_GetBucketTagging(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{ + "key": "val", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key", Value: "val"}, + }, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) { + return tt.input.beRes.(map[string]string), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketOwnershipControls(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: types.ObjectOwnership(""), + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{{}}, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: types.ObjectOwnershipBucketOwnerEnforced, + }, + output: testOutput{ + response: &Response{ + Data: s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{{ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced}}, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { + return tt.input.beRes.(types.ObjectOwnership), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketOwnershipControls, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketVersioning(t *testing.T) { + status := types.BucketVersioningStatusEnabled + validRes := s3response.GetBucketVersioningOutput{ + Status: &status, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "not admin or root", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + utils.ContextKeyPublicBucket: true, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.GetBucketVersioningOutput{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.GetBucketVersioningOutput{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) { + return tt.input.beRes.(s3response.GetBucketVersioningOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketVersioning, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketCors(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: []byte{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: []byte("mock_cors_resp"), + }, + output: testOutput{ + response: &Response{ + Data: []byte("mock_cors_resp"), + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketCorsFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketCors, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketPolicy(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrAccessDenied), + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: []byte{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: []byte("mock_policy_resp"), + }, + output: testOutput{ + response: &Response{ + Data: []byte("mock_policy_resp"), + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketPolicy, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListObjectVersions(t *testing.T) { + listVersionsResult := s3response.ListVersionsResult{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Versions: []s3response.ObjectVersion{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxKeys), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListVersionsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListVersionsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listVersionsResult, + }, + output: testOutput{ + response: &Response{ + Data: listVersionsResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) { + return tt.input.beRes.(s3response.ListVersionsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjectVersions, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectLockConfiguration(t *testing.T) { + cfgBytes, err := json.Marshal( + auth.BucketLockConfig{ + Enabled: true, + }) + assert.NoError(t, err) + + var lockCfg *types.ObjectLockConfiguration + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + }, + output: testOutput{ + response: &Response{ + Data: lockCfg, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse object lock config: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: cfgBytes, + }, + output: testOutput{ + response: &Response{ + Data: &types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + Rule: &types.ObjectLockRule{ + DefaultRetention: nil, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectLockConfiguration, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetBucketAcl(t *testing.T) { + aclBytes, err := json.Marshal( + auth.ACL{ + Owner: "root", + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{'d'}, + }, + output: testOutput{ + response: &Response{ + Data: auth.GetBucketAclOutput{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse acl: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: aclBytes, + }, + output: testOutput{ + response: &Response{ + Data: auth.GetBucketAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{ + Grants: []auth.Grant{}, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetBucketAcl, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListMultipartUploads(t *testing.T) { + listMpResult := s3response.ListMultipartUploadsResult{ + Prefix: "prefix", + Delimiter: "delim", + Bucket: "bucket", + Uploads: []s3response.Upload{ + {Key: "my-key"}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max uploads", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-uploads": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxUploads), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListMultipartUploadsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListMultipartUploadsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listMpResult, + }, + output: testOutput{ + response: &Response{ + Data: listMpResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + return tt.input.beRes.(s3response.ListMultipartUploadsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListMultipartUploads, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_ListObjectsV2(t *testing.T) { + listV2Result := s3response.ListObjectsV2Result{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Contents: []s3response.Object{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxKeys), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListObjectsV2Result{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListObjectsV2Result{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listV2Result, + }, + output: testOutput{ + response: &Response{ + Data: listV2Result, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + return tt.input.beRes.(s3response.ListObjectsV2Result), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjectsV2, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_ListObjects(t *testing.T) { + listResult := s3response.ListObjectsResult{ + Name: utils.GetStringPtr("name"), + Prefix: utils.GetStringPtr("prefix"), + Delimiter: utils.GetStringPtr("delim"), + Contents: []s3response.Object{ + {Key: utils.GetStringPtr("my-key")}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max keys", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-keys": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxKeys), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListObjectsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListObjectsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listResult, + }, + output: testOutput{ + response: &Response{ + Data: listResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return tt.input.beRes.(s3response.ListObjectsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListObjects, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} diff --git a/s3api/controllers/bucket-head_test.go b/s3api/controllers/bucket-head_test.go new file mode 100644 index 00000000..4b5a6e4d --- /dev/null +++ b/s3api/controllers/bucket-head_test.go @@ -0,0 +1,136 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func TestS3ApiController_HeadBucket(t *testing.T) { + region := "us-east-1" + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: false, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "user", + Role: auth.RoleUser, + }, + utils.ContextKeyRegion: region, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: region, + }, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + { + name: "successful response", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyRegion: region, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "X-Amz-Access-Point-Alias": utils.GetStringPtr("false"), + "X-Amz-Bucket-Region": utils.GetStringPtr(region), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + HeadBucketFunc: func(contextMoqParam context.Context, headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + return &s3.HeadBucketOutput{}, tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.HeadBucket, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go index 38bcb327..f990dd1e 100644 --- a/s3api/controllers/bucket-list.go +++ b/s3api/controllers/bucket-list.go @@ -29,7 +29,6 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { cToken := ctx.Query("continuation-token") prefix := ctx.Query("prefix") maxBucketsStr := ctx.Query("max-buckets") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) var maxBuckets int32 = 10000 diff --git a/s3api/controllers/bucket-list_test.go b/s3api/controllers/bucket-list_test.go new file mode 100644 index 00000000..7f6bcc79 --- /dev/null +++ b/s3api/controllers/bucket-list_test.go @@ -0,0 +1,112 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "testing" + + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_ListBuckets(t *testing.T) { + validRes := s3response.ListAllMyBucketsResult{ + Owner: s3response.CanonicalUser{ + ID: "root", + }, + Buckets: s3response.ListAllMyBucketsList{ + Bucket: []s3response.ListAllMyBucketsEntry{ + {Name: "test"}, + }, + }, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "invalid max buckets", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-buckets": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxBuckets), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + queries: map[string]string{ + "max-buckets": "3", + }, + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { + return tt.input.beRes.(s3response.ListAllMyBucketsResult), tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListBuckets, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + queries: tt.input.queries, + }) + }) + } +} diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go index 283afc7f..22e5f2f0 100644 --- a/s3api/controllers/bucket-post.go +++ b/s3api/controllers/bucket-post.go @@ -37,19 +37,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - var dObj s3response.DeleteObjects - - err := xml.Unmarshal(ctx.Body(), &dObj) - if err != nil { - debuglogger.Logf("error unmarshalling delete objects: %v", err) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -68,6 +56,17 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { }, err } + var dObj s3response.DeleteObjects + err = xml.Unmarshal(ctx.Body(), &dObj) + if err != nil { + debuglogger.Logf("error unmarshalling delete objects: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be) if err != nil { return &Response{ diff --git a/s3api/controllers/bucket-post_test.go b/s3api/controllers/bucket-post_test.go new file mode 100644 index 00000000..40cf2879 --- /dev/null +++ b/s3api/controllers/bucket-post_test.go @@ -0,0 +1,165 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/xml" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_DeleteObjects(t *testing.T) { + validBody, err := xml.Marshal(s3response.DeleteObjects{ + Objects: []types.ObjectIdentifier{ + {Key: utils.GetStringPtr("obj")}, + }, + }) + assert.NoError(t, err) + + validRes := s3response.DeleteResult{ + Deleted: []types.DeletedObject{ + {Key: utils.GetStringPtr("key")}, + }, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "check object access returns error", + input: testInput{ + locals: defaultLocals, + body: validBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.DeleteResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validBody, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: s3response.DeleteResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 1, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validBody, + beRes: validRes, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + }, + output: testOutput{ + response: &Response{ + Data: validRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRemovedDeleteObjects, + ObjectCount: 1, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteObjectsFunc: func(contextMoqParam context.Context, deleteObjectsInput *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { + return tt.input.beRes.(s3response.DeleteResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, tt.input.extraMockErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteObjects, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} diff --git a/s3api/controllers/bucket-put_test.go b/s3api/controllers/bucket-put_test.go new file mode 100644 index 00000000..622354ac --- /dev/null +++ b/s3api/controllers/bucket-put_test.go @@ -0,0 +1,813 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_PutBucketTagging(t *testing.T) { + validTaggingBody, err := xml.Marshal(s3response.TaggingInput{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + { + Key: "key", + Value: "val", + }, + }, + }, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validTaggingBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validTaggingBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutBucketTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_PutBucketOwnershipControls(t *testing.T) { + validOwnershipBody, err := xml.Marshal( + s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{ + {ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced}, + }, + }) + assert.NoError(t, err) + + invalidRuleCountBody, err := xml.Marshal( + s3response.OwnershipControls{ + Rules: []types.OwnershipControlsRule{ + {ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced}, + {ObjectOwnership: types.ObjectOwnershipBucketOwnerPreferred}, + }, + }, + ) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid rules count", + input: testInput{ + locals: defaultLocals, + body: invalidRuleCountBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend error", + input: testInput{ + locals: defaultLocals, + body: validOwnershipBody, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success", + input: testInput{ + locals: defaultLocals, + body: validOwnershipBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController(t, ctrl.PutBucketOwnershipControls, tt.output.response, tt.output.err, ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_PutBucketVersioning(t *testing.T) { + validVersioningBody, err := xml.Marshal( + types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + ) + assert.NoError(t, err) + + invalidVersioningStatusBody, err := xml.Marshal( + types.VersioningConfiguration{ + Status: types.BucketVersioningStatus("invalid_status"), + }, + ) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "invalid rules count", + input: testInput{ + locals: defaultLocals, + body: invalidVersioningStatusBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend error", + input: testInput{ + locals: defaultLocals, + body: validVersioningBody, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success", + input: testInput{ + locals: defaultLocals, + body: validVersioningBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController(t, ctrl.PutBucketVersioning, tt.output.response, tt.output.err, ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_PutObjectLockConfiguration(t *testing.T) { + validLockBody, err := xml.Marshal( + types.ObjectLockConfiguration{ + ObjectLockEnabled: types.ObjectLockEnabledEnabled, + }, + ) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend error", + input: testInput{ + locals: defaultLocals, + body: validLockBody, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success", + input: testInput{ + locals: defaultLocals, + body: validLockBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController(t, ctrl.PutObjectLockConfiguration, tt.output.response, tt.output.err, ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_PutBucketCors(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + }, + { + name: "success", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketCorsFunc: func(contextMoqParam context.Context, bytes []byte) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController(t, ctrl.PutBucketCors, tt.output.response, tt.output.err, ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_PutBucketPolicy(t *testing.T) { + validPolicyDocument := + `{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PublicReadGetObject", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::bucket/*" + } + ] + }` + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid policy document", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.APIError{ + Code: "MalformedPolicy", + Description: "Policies must be valid JSON and the first byte must be '{'", + HTTPStatusCode: http.StatusBadRequest, + }, + }, + }, + { + name: "backend error", + input: testInput{ + locals: defaultLocals, + body: []byte(validPolicyDocument), + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: "root"}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success", + input: testInput{ + locals: defaultLocals, + body: []byte(validPolicyDocument), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController(t, ctrl.PutBucketPolicy, tt.output.response, tt.output.err, ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_CreateBucket(t *testing.T) { + adminAcc := auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + } + userAcc := auth.Account{ + Access: "user", + Role: auth.RoleUser, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "access denied", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: userAcc, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid bucket name", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + bucket: "invalid_bucket_name", + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidBucketName), + }, + }, + { + name: "invalid ownership", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + headers: map[string]string{ + "X-Amz-Object-Ownership": "invalid_ownership", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.APIError{ + Code: "InvalidArgument", + Description: "Invalid x-amz-object-ownership header: invalid_ownership", + HTTPStatusCode: http.StatusBadRequest, + }, + }, + }, + { + name: "invalid ownership + acl", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + headers: map[string]string{ + "X-Amz-Object-Ownership": string(types.ObjectOwnershipBucketOwnerEnforced), + "X-Amz-Acl": string(types.BucketCannedACLPublicRead), + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership), + }, + }, + { + name: "both grants and canned acl", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + headers: map[string]string{ + "X-Amz-Acl": string(types.BucketCannedACLPublicRead), + "X-Amz-Grant-Read": userAcc.Access, + "X-Amz-Object-Ownership": string(types.ObjectOwnershipBucketOwnerPreferred), + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access}, + }, + err: s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), + }, + }, + { + name: "fail to update the acl", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + headers: map[string]string{ + "X-Amz-Grant-Read": userAcc.Access, + "X-Amz-Object-Ownership": string(types.ObjectOwnershipBucketOwnerPreferred), + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access}, + }, + err: fmt.Errorf("accounts does not exist: %s", userAcc.Access), + }, + }, + { + name: "backend error", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{BucketOwner: adminAcc.Access}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyAccount: adminAcc, + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: adminAcc.Access, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error { + return tt.input.beErr + }, + } + + ctrl := S3ApiController{ + be: be, + iam: auth.NewIAMServiceSingle(adminAcc), + } + + testController(t, ctrl.CreateBucket, tt.output.response, tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + bucket: tt.input.bucket, + headers: tt.input.headers, + }) + }) + } +} + +// TODO: add a test for PutBucketAcl diff --git a/tests/integration/tests.go b/tests/integration/tests.go index d14b40e6..7872b5d6 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -1297,11 +1297,6 @@ func PresignedAuth_UploadPart(s *S3Conf) error { return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { key, partNumber := "my-mp", int32(1) - err := setup(s, bucket) - if err != nil { - return err - } - clt := s.GetClient() mp, err := createMp(clt, bucket, key) if err != nil { @@ -1351,11 +1346,6 @@ func PresignedAuth_UploadPart(s *S3Conf) error { return fmt.Errorf("expected uploaded part part-number to be %v, instead got %v", partNumber, *out.Parts[0].PartNumber) } - err = teardown(s, bucket) - if err != nil { - return err - } - return nil }) } From 67d0750ee08de65dfd1d5420454a20a1446f49ae Mon Sep 17 00:00:00 2001 From: niksis02 Date: Tue, 8 Jul 2025 22:20:00 +0400 Subject: [PATCH 14/20] feat: adds unit tests for object DELETE and POST operations --- s3api/controllers/object-delete_test.go | 296 +++++++++++++ s3api/controllers/object-post.go | 55 ++- s3api/controllers/object-post_test.go | 561 ++++++++++++++++++++++++ s3response/s3response.go | 4 + 4 files changed, 887 insertions(+), 29 deletions(-) create mode 100644 s3api/controllers/object-delete_test.go create mode 100644 s3api/controllers/object-post_test.go diff --git a/s3api/controllers/object-delete_test.go b/s3api/controllers/object-delete_test.go new file mode 100644 index 00000000..d9054351 --- /dev/null +++ b/s3api/controllers/object-delete_test.go @@ -0,0 +1,296 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "net/http" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" +) + +func TestS3ApiController_DeleteObjectTagging(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + EventName: s3event.EventObjectTaggingDelete, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + EventName: s3event.EventObjectTaggingDelete, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteObjectTaggingFunc: func(contextMoqParam context.Context, bucket, object string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteObjectTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_AbortMultipartUpload(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + AbortMultipartUploadFunc: func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.AbortMultipartUpload, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_DeleteObject(t *testing.T) { + delMarker, versionId := true, "versionId" + var emptyRes *s3.DeleteObjectOutput + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "object locked", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLocked), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + beRes: emptyRes, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + EventName: s3event.EventObjectRemovedDelete, + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + beRes: &s3.DeleteObjectOutput{ + DeleteMarker: &delMarker, + VersionId: &versionId, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "x-amz-version-id": &versionId, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusNoContent, + EventName: s3event.EventObjectRemovedDelete, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + return tt.input.beRes.(*s3.DeleteObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, tt.input.extraMockErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.DeleteObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 0d6fe8cb..658320a4 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -39,15 +39,6 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var restoreRequest types.RestoreRequest - if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil { - debuglogger.Logf("failed to parse the request body: %v", err) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrMalformedXML) - } err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -68,6 +59,16 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { }, err } + var restoreRequest types.RestoreRequest + if err := xml.Unmarshal(ctx.Body(), &restoreRequest); err != nil { + debuglogger.Logf("failed to parse the request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + err = c.be.RestoreObject(ctx.Context(), &s3.RestoreObjectInput{ Bucket: &bucket, Key: &key, @@ -89,19 +90,7 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var payload s3response.SelectObjectContentPayload - - err := xml.Unmarshal(ctx.Body(), &payload) - if err != nil { - debuglogger.Logf("error unmarshalling select object content: %v", err) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrMalformedXML) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -121,6 +110,17 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) }, err } + var payload s3response.SelectObjectContentPayload + err = xml.Unmarshal(ctx.Body(), &payload) + if err != nil { + debuglogger.Logf("error unmarshalling select object content: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + sw := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{ Bucket: &bucket, @@ -260,11 +260,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err }, err } - data := struct { - Parts []types.CompletedPart `xml:"Part"` - }{} - - err = xml.Unmarshal(ctx.Body(), &data) + var body s3response.CompleteMultipartUploadRequestBody + err = xml.Unmarshal(ctx.Body(), &body) if err != nil { debuglogger.Logf("error unmarshalling complete multipart upload: %v", err) return &Response{ @@ -274,7 +271,7 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err }, s3err.GetAPIError(s3err.ErrMalformedXML) } - if len(data.Parts) == 0 { + if len(body.Parts) == 0 { debuglogger.Logf("empty parts provided for complete multipart upload") return &Response{ MetaOpts: &MetaOptions{ @@ -332,7 +329,7 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err Key: &key, UploadId: &uploadId, MultipartUpload: &types.CompletedMultipartUpload{ - Parts: data.Parts, + Parts: body.Parts, }, MpuObjectSize: mpuObjectSize, ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), diff --git a/s3api/controllers/object-post_test.go b/s3api/controllers/object-post_test.go new file mode 100644 index 00000000..baca9f3e --- /dev/null +++ b/s3api/controllers/object-post_test.go @@ -0,0 +1,561 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "bufio" + "context" + "encoding/xml" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_RestoreObject(t *testing.T) { + validRestoreBody, err := xml.Marshal(types.RestoreRequest{ + Description: utils.GetStringPtr("description"), + Type: types.RestoreRequestTypeSelect, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validRestoreBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRestoreCompleted, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validRestoreBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectRestoreCompleted, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.RestoreObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_SelectObjectContent(t *testing.T) { + validSelectBody, err := xml.Marshal(s3response.SelectObjectContentPayload{ + Expression: utils.GetStringPtr("expression"), + ExpressionType: types.ExpressionTypeSql, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validSelectBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) { + return func(w *bufio.Writer) {} + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.SelectObjectContent, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_CreateMultipartUpload(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid object lock headers", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Object-Lock-Mode": string(types.ObjectLockModeGovernance), + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders), + }, + }, + { + name: "invalid checksum headers", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Checksum-Algorithm": "invalid_checksum_algo", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.InitiateMultipartUploadResult{}, + }, + output: testOutput{ + response: &Response{ + Data: s3response.InitiateMultipartUploadResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: s3response.InitiateMultipartUploadResult{}, + headers: map[string]string{ + "x-amz-checksum-algorithm": string(types.ChecksumAlgorithmCrc32), + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.InitiateMultipartUploadResult{}, + Headers: map[string]*string{ + "x-amz-checksum-algorithm": utils.ConvertToStringPtr(types.ChecksumAlgorithmCrc32), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { + return tt.input.beRes.(s3response.InitiateMultipartUploadResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.CreateMultipartUpload, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + }) + }) + } +} + +func TestS3ApiController_CompleteMultipartUpload(t *testing.T) { + emptyMpPartsBody, err := xml.Marshal(s3response.CompleteMultipartUploadRequestBody{ + Parts: []types.CompletedPart{}, + }) + assert.NoError(t, err) + pn := int32(1) + + validMpBody, err := xml.Marshal(s3response.CompleteMultipartUploadRequestBody{ + Parts: []types.CompletedPart{ + { + PartNumber: &pn, + ETag: utils.GetStringPtr("ETag"), + }, + }, + }) + assert.NoError(t, err) + + versionId, ETag := "versionId", "mock-ETag" + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "request body empty mp parts", + input: testInput{ + locals: defaultLocals, + body: emptyMpPartsBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrEmptyParts), + }, + }, + { + name: "invalid mp parts header string", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + headers: map[string]string{ + "X-Amz-Mp-Object-Size": "invalid_mp_object_size", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "negative mp parts header value", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + headers: map[string]string{ + "X-Amz-Mp-Object-Size": "-4", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidMpObjectSizeErr(-4), + }, + }, + { + name: "invalid checksum headers", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + headers: map[string]string{ + "X-Amz-Sdk-Checksum-Algorithm": "invalid_checksum_algo", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm), + }, + }, + { + name: "invalid checksum type", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + headers: map[string]string{ + "X-Amz-Checksum-Type": "invalid_checksum_type", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.CompleteMultipartUploadResult{}, + }, + output: testOutput{ + response: &Response{ + Data: s3response.CompleteMultipartUploadResult{}, + Headers: map[string]*string{ + "x-amz-version-id": &versionId, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventCompleteMultipartUpload, + VersionId: &versionId, + ObjectETag: nil, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validMpBody, + beRes: s3response.CompleteMultipartUploadResult{ + ETag: &ETag, + }, + headers: map[string]string{ + "X-Amz-Mp-Object-Size": "3", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.CompleteMultipartUploadResult{ + ETag: &ETag, + }, + Headers: map[string]*string{ + "x-amz-version-id": &versionId, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventCompleteMultipartUpload, + VersionId: &versionId, + ObjectETag: &ETag, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { + return tt.input.beRes.(s3response.CompleteMultipartUploadResult), versionId, tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.CompleteMultipartUpload, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + }) + }) + } +} diff --git a/s3response/s3response.go b/s3response/s3response.go index 810f0b74..223e6f0b 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -412,6 +412,10 @@ func (r CopyPartResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error return e.EncodeElement(aux, start) } +type CompleteMultipartUploadRequestBody struct { + Parts []types.CompletedPart `xml:"Part"` +} + type CompleteMultipartUploadResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` Location *string From ba76aea17a1e90da1f2ae4bab42bb22e36a8e6bb Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 10 Jul 2025 03:12:52 +0400 Subject: [PATCH 15/20] feat: adds unit tests for the object HEAD and GET controllers. --- s3api/controllers/object-get.go | 38 +- s3api/controllers/object-get_test.go | 820 ++++++++++++++++++++++++++ s3api/controllers/object-head.go | 28 +- s3api/controllers/object-head_test.go | 183 ++++++ s3api/utils/utils.go | 6 + 5 files changed, 1042 insertions(+), 33 deletions(-) create mode 100644 s3api/controllers/object-get_test.go create mode 100644 s3api/controllers/object-head_test.go diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index 98d97917..b86bffc3 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -220,6 +220,25 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.ListMultipartUploadPartsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + // parse the part number marker if partNumberMarker != "" { n, err := strconv.Atoi(partNumberMarker) @@ -247,25 +266,6 @@ func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrInvalidMaxParts) } - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.ListMultipartUploadPartsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{ Bucket: &bucket, Key: &key, diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go new file mode 100644 index 00000000..5f3f0a0f --- /dev/null +++ b/s3api/controllers/object-get_test.go @@ -0,0 +1,820 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_GetObjectTagging(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: map[string]string{ + "key": "val", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.Tagging{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + {Key: "key", Value: "val"}, + }, + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectTaggingFunc: func(contextMoqParam context.Context, bucket, object string) (map[string]string, error) { + return tt.input.beRes.(map[string]string), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetObjectRetention(t *testing.T) { + retBytes, err := json.Marshal(types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + }) + assert.NoError(t, err) + + var retention *types.ObjectLockRetention + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "invalid data from backend", + input: testInput{ + locals: defaultLocals, + beRes: []byte{}, + }, + output: testOutput{ + response: &Response{ + Data: retention, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: fmt.Errorf("parse object lock retention: "), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: retBytes, + }, + output: testOutput{ + response: &Response{ + Data: &types.ObjectLockRetention{ + Mode: types.ObjectLockRetentionModeCompliance, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) { + return tt.input.beRes.([]byte), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectRetention, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetObjectLegalHold(t *testing.T) { + var legalHold *bool + var emptyLegalHold *s3response.GetObjectLegalHoldResult + status := true + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: legalHold, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: emptyLegalHold, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: &status, + }, + output: testOutput{ + response: &Response{ + Data: &s3response.GetObjectLegalHoldResult{ + Status: types.ObjectLockLegalHoldStatusOn, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) { + return tt.input.beRes.(*bool), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectLegalHold, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_GetObjectAcl(t *testing.T) { + var emptyRes *s3.GetObjectAclOutput + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: emptyRes, + beErr: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + output: testOutput{ + response: &Response{ + Data: emptyRes, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: &s3.GetObjectAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("something"), + }, + }, + }, + output: testOutput{ + response: &Response{ + Data: &s3.GetObjectAclOutput{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("something"), + }, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { + return tt.input.beRes.(*s3.GetObjectAclOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectAcl, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_ListParts(t *testing.T) { + listPartsResult := s3response.ListPartsResult{ + Bucket: "my-bucket", + Key: "obj", + IsTruncated: false, + Parts: []s3response.Part{ + {ETag: "ETag"}, + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid part number marker", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "part-number-marker": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), + }, + }, + { + name: "invalid max parts", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "max-parts": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxParts), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: s3response.ListPartsResult{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListPartsResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: listPartsResult, + }, + output: testOutput{ + response: &Response{ + Data: listPartsResult, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) { + return tt.input.beRes.(s3response.ListPartsResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.ListParts, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_GetObjectAttributes(t *testing.T) { + delMarker, lastModTime, etag := true, time.Now(), "ETag" + timeFormatted := lastModTime.UTC().Format(iso8601TimeFormatExtended) + + validRes := s3response.GetObjectAttributesResponse{ + DeleteMarker: &delMarker, + LastModified: &lastModTime, + VersionId: utils.GetStringPtr("versionId"), + ETag: &etag, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid max parts", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Max-Parts": "-1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMaxParts), + }, + }, + { + name: "invalid object attributes", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Object-Attributes": "invalid_attribute", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidObjectAttributes), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + headers: map[string]string{ + "X-Amz-Object-Attributes": "ETag", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr("versionId"), + "x-amz-delete-marker": utils.GetStringPtr("true"), + }, + Data: nil, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + beRes: validRes, + headers: map[string]string{ + "X-Amz-Object-Attributes": "ETag", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr("versionId"), + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": &timeFormatted, + }, + Data: s3response.GetObjectAttributesResponse{ + ETag: &etag, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) { + return tt.input.beRes.(s3response.GetObjectAttributesResponse), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObjectAttributes, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + }) + }) + } +} + +func TestS3ApiController_GetObject(t *testing.T) { + tm := time.Now() + cLength := int64(11) + rdr := io.NopCloser(strings.NewReader("hello world")) + delMarker := true + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid checksum mode", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "x-amz-checksum-mode": "invalid_checksum_mode", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + beRes: &s3.GetObjectOutput{ + DeleteMarker: &delMarker, + LastModified: &tm, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(tm.UTC().Format(timefmt)), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + // TODO: add a test case for overflowing content-length + // simulate a 32 bit arch to test the case + { + name: "successful response", + input: testInput{ + headers: map[string]string{ + "Range": "100-200", + }, + queries: map[string]string{ + "versionId": "versionId", + }, + locals: defaultLocals, + beRes: &s3.GetObjectOutput{ + ETag: utils.GetStringPtr("ETag"), + ContentType: utils.GetStringPtr("application/xml"), + ContentLength: &cLength, + Body: rdr, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": nil, + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "x-amz-tagging-count": nil, + "Content-Type": utils.GetStringPtr("application/xml"), + "Content-Length": utils.GetStringPtr("11"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + Status: http.StatusPartialContent, + ContentLength: cLength, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return tt.input.beRes.(*s3.GetObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.GetObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index 2eb84d46..5b0aa304 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -41,20 +41,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { objRange := ctx.Get("Range") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) - var partNumber *int32 - if ctx.Request().URI().QueryArgs().Has("partNumber") { - if partNumberQuery < 1 || partNumberQuery > 10000 { - debuglogger.Logf("invalid part number: %d", partNumberQuery) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) - } - - partNumber = &partNumberQuery - } - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -75,6 +61,20 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { }, err } + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < 1 || partNumberQuery > 10000 { + debuglogger.Logf("invalid part number: %d", partNumberQuery) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + partNumber = &partNumberQuery + } + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) diff --git a/s3api/controllers/object-head_test.go b/s3api/controllers/object-head_test.go new file mode 100644 index 00000000..9d243e33 --- /dev/null +++ b/s3api/controllers/object-head_test.go @@ -0,0 +1,183 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func TestS3ApiController_HeadObject(t *testing.T) { + tm := time.Now() + cLength := int64(100) + + failingBeRes := &s3.HeadObjectOutput{ + LastModified: &tm, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid part number", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "-4", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidPartNumber), + }, + }, + { + name: "invalid checksum mode", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "x-amz-checksum-mode": "invalid_checksum_mode", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + beRes: failingBeRes, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(tm.UTC().Format(timefmt)), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), + }, + }, + { + name: "successful response", + input: testInput{ + queries: map[string]string{ + "partNumber": "4", + }, + locals: defaultLocals, + beRes: &s3.HeadObjectOutput{ + ETag: utils.GetStringPtr("ETag"), + ContentType: utils.GetStringPtr("application/xml"), + ContentLength: &cLength, + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-restore": nil, + "accept-ranges": nil, + "Content-Range": nil, + "Content-Disposition": nil, + "Content-Encoding": nil, + "Content-Language": nil, + "Cache-Control": nil, + "Expires": nil, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-version-id": nil, + "x-amz-mp-parts-count": nil, + "x-amz-object-lock-mode": nil, + "x-amz-object-lock-legal-hold": nil, + "x-amz-storage-class": nil, + "x-amz-checksum-type": nil, + "x-amz-object-lock-retain-until-date": nil, + "Last-Modified": nil, + "Content-Type": utils.GetStringPtr("application/xml"), + "Content-Length": utils.GetStringPtr("100"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + HeadObjectFunc: func(contextMoqParam context.Context, headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + return tt.input.beRes.(*s3.HeadObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.HeadObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + queries: tt.input.queries, + headers: tt.input.headers, + }) + }) + } +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index bf70fbf4..5489d9c5 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -677,6 +677,9 @@ func GetStringPtr(str string) *string { // Converts any type to a string pointer func ConvertToStringPtr[T any](val T) *string { str := fmt.Sprint(val) + if str == "" { + return nil + } return &str } @@ -694,6 +697,9 @@ func FormatDatePtrToString(date *time.Time, format string) *string { if date == nil { return nil } + if date.IsZero() { + return nil + } formatted := date.UTC().Format(format) return &formatted From 7f9ab35347907372583183f1709bc5fda728b2ef Mon Sep 17 00:00:00 2001 From: niksis02 Date: Fri, 11 Jul 2025 03:45:06 +0400 Subject: [PATCH 16/20] feat: implements unit tests for object PUT controllers --- auth/bucket_policy.go | 3 +- s3api/controllers/base_test.go | 18 +- s3api/controllers/object-put.go | 192 ++-- s3api/controllers/object-put_test.go | 1243 ++++++++++++++++++++++++++ 4 files changed, 1310 insertions(+), 146 deletions(-) create mode 100644 s3api/controllers/object-put_test.go diff --git a/auth/bucket_policy.go b/auth/bucket_policy.go index 4f817199..6d6d1495 100644 --- a/auth/bucket_policy.go +++ b/auth/bucket_policy.go @@ -17,6 +17,7 @@ package auth import ( "encoding/json" "errors" + "fmt" "net/http" "github.com/versity/versitygw/s3err" @@ -194,7 +195,7 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error { var bucketPolicy BucketPolicy if err := json.Unmarshal(policy, &bucketPolicy); err != nil { - return err + return fmt.Errorf("failed to parse the bucket policy: %w", err) } resource := bucket diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 4f0173d6..5cabf374 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -54,15 +54,15 @@ var ( ) type testInput struct { - bucket string - object string - body []byte - locals map[utils.ContextKey]any - headers map[string]string - queries map[string]string - beRes any - beErr error - extraMockErr error + bucket string + body []byte + locals map[utils.ContextKey]any + headers map[string]string + queries map[string]string + beRes any + beErr error + extraMockErr error + extraMockResp any } type testOutput struct { diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 453f624f..124db9a5 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -42,16 +42,7 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) - if err != nil { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, AclPermission: auth.PermissionWrite, @@ -70,6 +61,15 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { }, err } + tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + err = c.be.PutObjectTagging(ctx.Context(), bucket, key, tagging) return &Response{ MetaOpts: &MetaOptions{ @@ -145,6 +145,24 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectLegalHoldAction, + IsBucketPublic: IsBucketPublic, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + var legalHold types.ObjectLockLegalHold if err := xml.Unmarshal(ctx.Body(), &legalHold); err != nil { debuglogger.Logf("failed to parse request body: %v", err) @@ -164,24 +182,6 @@ func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrMalformedXML) } - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.PutObjectLegalHoldAction, - IsBucketPublic: IsBucketPublic, - }); err != nil { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - err := c.be.PutObjectLegalHold(ctx.Context(), bucket, key, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) return &Response{ MetaOpts: &MetaOptions{ @@ -212,15 +212,6 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { contentLengthStr = decodedLength } - if partNumber < 1 || partNumber > 10000 { - debuglogger.Logf("invalid part number: %d", partNumber) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) - } - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, @@ -241,6 +232,15 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { }, err } + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) @@ -329,15 +329,6 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrInvalidCopySource) } - if partNumber < 1 || partNumber > 10000 { - debuglogger.Logf("invalid part number: %d", partNumber) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) - } - err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, auth.AccessOptions{ Acl: parsedAcl, @@ -357,6 +348,15 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { }, err } + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{ Bucket: &bucket, @@ -390,7 +390,6 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") granWrite := ctx.Get("X-Amz-Grant-Write") grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) @@ -415,95 +414,16 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { }, err } - var input *s3.PutObjectAclInput - if len(ctx.Body()) > 0 { - if grants+acl != "" { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) - } - - var accessControlPolicy auth.AccessControlPolicy - err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) - if err != nil { - debuglogger.Logf("error unmarshalling access control policy: %v", err) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) - } - - //TODO: This part will be changed when object acls are implemented - - grants := []types.Grant{} - for _, grt := range accessControlPolicy.AccessControlList.Grants { - grants = append(grants, types.Grant{ - Grantee: &types.Grantee{ - ID: &grt.Grantee.ID, - Type: grt.Grantee.Type, - }, - Permission: types.Permission(grt.Permission), - }) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &key, - ACL: "", - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: accessControlPolicy.Owner, - Grants: grants, - }, - } - } - if acl != "" { - if acl != "private" && acl != "public-read" && acl != "public-read-write" { - debuglogger.Logf("invalid acl: %q", acl) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) - } - if len(ctx.Body()) > 0 || grants != "" { - debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", grants, acl, len(ctx.Body())) - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &key, - ACL: types.ObjectCannedACL(acl), - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &parsedAcl.Owner}, - }, - } - } - if grants != "" { - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &key, - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &parsedAcl.Owner}, - }, - ACL: "", - } - } - - err = c.be.PutObjectAcl(ctx.Context(), input) + err = c.be.PutObjectAcl(ctx.Context(), &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantWrite: &granWrite, + ACL: types.ObjectCannedACL(acl), + GrantReadACP: &grantReadACP, + GrantWriteACP: &grantWriteACP, + }) return &Response{ MetaOpts: &MetaOptions{ BucketOwner: parsedAcl.Owner, diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go new file mode 100644 index 00000000..516f02dc --- /dev/null +++ b/s3api/controllers/object-put_test.go @@ -0,0 +1,1243 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/xml" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func TestS3ApiController_PutObjectTagging(t *testing.T) { + validTaggingBody, err := xml.Marshal( + s3response.TaggingInput{ + TagSet: s3response.TagSet{ + Tags: []s3response.Tag{ + { + Key: "key", + Value: "val", + }, + }, + }, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validTaggingBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectTaggingPut, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validTaggingBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectTaggingPut, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectTaggingFunc: func(contextMoqParam context.Context, bucket, object string, tags map[string]string) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutObjectTagging, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_PutObjectRetention(t *testing.T) { + retDate := time.Now().Add(time.Hour * 3) + validRetentionBody, err := xml.Marshal( + s3response.PutObjectRetentionInput{ + Mode: types.ObjectLockRetentionModeGovernance, + RetainUntilDate: s3response.AmzDate{ + Time: retDate, + }, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validRetentionBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "success bypass GetBucketPolicy fails", + input: testInput{ + locals: defaultLocals, + body: validRetentionBody, + extraMockErr: s3err.GetAPIError(s3err.ErrAccessDenied), + headers: map[string]string{ + "X-Amz-Bypass-Governance-Retention": "true", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "success bypass VerifyBucketPolicy fails", + input: testInput{ + locals: defaultLocals, + body: validRetentionBody, + extraMockResp: []byte("invalid_policy"), + headers: map[string]string{ + "X-Amz-Bypass-Governance-Retention": "true", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validRetentionBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + if tt.input.extraMockResp == nil { + return nil, tt.input.extraMockErr + } else { + return tt.input.extraMockResp.([]byte), tt.input.extraMockErr + } + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutObjectRetention, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + }) + }) + } +} + +func TestS3ApiController_PutObjectLegalHold(t *testing.T) { + validBody, err := xml.Marshal( + types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatusOn, + }) + assert.NoError(t, err) + invalidStatusBody, err := xml.Marshal( + types.ObjectLockLegalHold{ + Status: types.ObjectLockLegalHoldStatus("invalid_status"), + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid legal hold status", + input: testInput{ + locals: defaultLocals, + body: invalidStatusBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + body: validBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + body: validBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutObjectLegalHold, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + }) + }) + } +} + +func TestS3ApiController_UploadPart(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid part number", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "-2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidPartNumber), + }, + }, + { + name: "invalid content length", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Decoded-Content-Length": "invalid_cLength", + }, + queries: map[string]string{ + "partNumber": "2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "invalid checksum header", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Sdk-Checksum-Algorithm": "invalid_algo", + }, + queries: map[string]string{ + "partNumber": "2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: &s3.UploadPartOutput{}, + queries: map[string]string{ + "partNumber": "2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyBodyReader: strings.NewReader("hello world"), + }, + queries: map[string]string{ + "partNumber": "2", + }, + headers: map[string]string{ + "Content-Length": "4", + }, + body: []byte("bbbb"), + beRes: &s3.UploadPartOutput{ + ETag: utils.GetStringPtr("ETag"), + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ContentLength: 4, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + return tt.input.beRes.(*s3.UploadPartOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.UploadPart, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_UploadPartCopy(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid copy source", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bad%G1", + }, + queries: map[string]string{ + "partNumber": "2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidCopySource), + }, + }, + { + name: "invalid part number", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + queries: map[string]string{ + "partNumber": "-2", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidPartNumber), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.CopyPartResult{}, + queries: map[string]string{ + "partNumber": "2", + }, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.CopyPartResult{}, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + queries: map[string]string{ + "partNumber": "2", + }, + + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + beRes: s3response.CopyPartResult{ + CopySourceVersionId: "versionId", + }, + }, + output: testOutput{ + response: &Response{ + Data: s3response.CopyPartResult{ + CopySourceVersionId: "versionId", + }, + Headers: map[string]*string{ + "x-amz-copy-source-version-id": utils.GetStringPtr("versionId"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { + return tt.input.beRes.(s3response.CopyPartResult), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.UploadPartCopy, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + headers: tt.input.headers, + queries: tt.input.queries, + }) + }) + } +} + +func TestS3ApiController_PutObjectAcl(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectAclPut, + }, + }, + err: s3err.GetAPIError(s3err.ErrNotImplemented), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectAclPut, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error { + return tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutObjectAcl, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + }) + }) + } +} + +func TestS3ApiController_CopyObject(t *testing.T) { + var nilResp *s3response.CopyObjectResult + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "invalid copy source", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bad%G1", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidCopySource), + }, + }, + { + name: "invalid copy source modified since", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "X-Amz-Copy-Source-If-Modified-Since": "invalid_date", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidCopySource), + }, + }, + { + name: "invalid copy source unmodified since", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidCopySource), + }, + }, + { + name: "invalid metadata directive", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "X-Amz-Copy-Source-If-Unmodified-Since": "20250102T150405Z", + "X-Amz-Copy-Source-If-Modified-Since": "20240102T150405Z", + "X-Amz-Metadata-Directive": "invalid_metadat_directive", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidMetadataDirective), + }, + }, + { + name: "invalid tagging directive", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "X-Amz-Tagging-Directive": "invalid_tagging_directive", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidTaggingDirective), + }, + }, + { + name: "invalid checksum algorithm", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "x-amz-checksum-algorithm": "invalid_checksum_algo", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm), + }, + }, + { + name: "invalid object lock headers", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + "X-Amz-Object-Lock-Mode": "GOVERNANCE", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.CopyObjectOutput{}, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + }, + output: testOutput{ + response: &Response{ + Data: nilResp, + Headers: map[string]*string{ + "x-amz-copy-source-version-id": nil, + "x-amz-version-id": nil, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectCreatedCopy, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: defaultLocals, + headers: map[string]string{ + "X-Amz-Copy-Source": "bucket/object", + }, + beRes: s3response.CopyObjectOutput{ + CopySourceVersionId: utils.GetStringPtr("copySourceVersionId"), + VersionId: utils.GetStringPtr("versionId"), + CopyObjectResult: &s3response.CopyObjectResult{ + ETag: utils.GetStringPtr("ETag"), + }, + }, + }, + output: testOutput{ + response: &Response{ + Data: &s3response.CopyObjectResult{ + ETag: utils.GetStringPtr("ETag"), + }, + Headers: map[string]*string{ + "x-amz-copy-source-version-id": utils.GetStringPtr("copySourceVersionId"), + "x-amz-version-id": utils.GetStringPtr("versionId"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ObjectETag: utils.GetStringPtr("ETag"), + EventName: s3event.EventObjectCreatedCopy, + VersionId: utils.GetStringPtr("versionId"), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) { + return tt.input.beRes.(s3response.CopyObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.CopyObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + headers: tt.input.headers, + }) + }) + } +} + +func TestS3ApiController_PutObject(t *testing.T) { + str := "" + emptyStringPtr := &str + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "verify access fails", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "locked object", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "invalid content length", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + headers: map[string]string{ + "X-Amz-Decoded-Content-Length": "invalid_length", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "invalid object lock headers", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + headers: map[string]string{ + "X-Amz-Object-Lock-Mode": "GOVERNANCE", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders), + }, + }, + { + name: "invalid checksum headers", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + headers: map[string]string{ + "X-Amz-Sdk-Checksum-Algorithm": "invalid_algo", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm), + }, + }, + { + name: "backend returns error", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + beRes: s3response.PutObjectOutput{}, + body: []byte("aaa"), + headers: map[string]string{ + "Content-Length": "3", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": emptyStringPtr, + "x-amz-checksum-crc32": nil, + "x-amz-checksum-crc32c": nil, + "x-amz-checksum-crc64nvme": nil, + "x-amz-checksum-sha1": nil, + "x-amz-checksum-sha256": nil, + "x-amz-checksum-type": nil, + "x-amz-version-id": emptyStringPtr, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + EventName: s3event.EventObjectCreatedPut, + ContentLength: 3, + ObjectSize: 3, + ObjectETag: emptyStringPtr, + }, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + locals: map[utils.ContextKey]any{ + utils.ContextKeyIsRoot: true, + utils.ContextKeyParsedAcl: auth.ACL{ + Owner: "root", + }, + utils.ContextKeyAccount: auth.Account{ + Access: "root", + Role: auth.RoleAdmin, + }, + utils.ContextKeyBodyReader: strings.NewReader("something"), + }, + extraMockErr: s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound), + headers: map[string]string{ + "Content-Length": "3", + }, + body: []byte("aaa"), + beRes: s3response.PutObjectOutput{ + ETag: "ETag", + ChecksumCRC32: utils.GetStringPtr("crc32"), + ChecksumCRC32C: utils.GetStringPtr("crc32c"), + ChecksumSHA1: utils.GetStringPtr("sha1"), + ChecksumSHA256: utils.GetStringPtr("sha256"), + ChecksumCRC64NVME: utils.GetStringPtr("crc64nvme"), + ChecksumType: types.ChecksumTypeComposite, + VersionID: "versionId", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "ETag": utils.GetStringPtr("ETag"), + "x-amz-checksum-crc32": utils.GetStringPtr("crc32"), + "x-amz-checksum-crc32c": utils.GetStringPtr("crc32c"), + "x-amz-checksum-crc64nvme": utils.GetStringPtr("crc64nvme"), + "x-amz-checksum-sha1": utils.GetStringPtr("sha1"), + "x-amz-checksum-sha256": utils.GetStringPtr("sha256"), + "x-amz-checksum-type": utils.GetStringPtr(string(types.ChecksumTypeComposite)), + "x-amz-version-id": utils.GetStringPtr("versionId"), + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + ObjectETag: utils.GetStringPtr("ETag"), + EventName: s3event.EventObjectCreatedPut, + ContentLength: 3, + ObjectSize: 3, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutObjectFunc: func(contextMoqParam context.Context, putObjectInput s3response.PutObjectInput) (s3response.PutObjectOutput, error) { + return tt.input.beRes.(s3response.PutObjectOutput), tt.input.beErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, tt.input.extraMockErr + }, + } + + ctrl := S3ApiController{ + be: be, + } + + testController( + t, + ctrl.PutObject, + tt.output.response, + tt.output.err, + ctxInputs{ + locals: tt.input.locals, + headers: tt.input.headers, + body: tt.input.body, + }) + }) + } +} From ab571a65710bbbb11e8e273a0c5188023ddd3161 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Sat, 12 Jul 2025 02:00:36 +0400 Subject: [PATCH 17/20] feat: implements unit tests for admin controllers --- s3api/controllers/admin.go | 5 + s3api/controllers/admin_test.go | 552 ++++++++++++++++++++++++++++++++ 2 files changed, 557 insertions(+) create mode 100644 s3api/controllers/admin_test.go diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index cbf19427..c4f5f18e 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -113,6 +113,11 @@ func (c AdminController) UpdateUser(ctx *fiber.Ctx) (*Response, error) { func (c AdminController) DeleteUser(ctx *fiber.Ctx) (*Response, error) { access := ctx.Query("access") + if access == "" { + return &Response{ + MetaOpts: &MetaOptions{}, + }, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess) + } err := c.iam.DeleteUserAccount(access) return &Response{ diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go new file mode 100644 index 00000000..44696200 --- /dev/null +++ b/s3api/controllers/admin_test.go @@ -0,0 +1,552 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "context" + "encoding/xml" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func TestAdminController_CreateUser(t *testing.T) { + validBody, err := xml.Marshal(auth.Account{ + Access: "access", + Secret: "secret", + Role: auth.RoleAdmin, + }) + assert.NoError(t, err) + + invalidUserRoleBody, err := xml.Marshal(auth.Account{ + Access: "access", + Secret: "secret", + Role: auth.Role("invalid_role"), + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "invalid request body", + input: testInput{ + body: []byte("invalid_request_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid user role", + input: testInput{ + body: invalidUserRoleBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminInvalidUserRole), + }, + }, + { + name: "backend returns user exists error", + input: testInput{ + body: validBody, + beErr: auth.ErrUserExists, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminUserExists), + }, + }, + { + name: "backend returns other error", + input: testInput{ + body: validBody, + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + body: validBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + Status: http.StatusCreated, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iam := &IAMServiceMock{ + CreateAccountFunc: func(account auth.Account) error { + return tt.input.beErr + }, + } + + ctrl := AdminController{ + iam: iam, + } + + testController( + t, + ctrl.CreateUser, + tt.output.response, + tt.output.err, + ctxInputs{ + body: tt.input.body, + }) + }) + } +} + +func TestAdminController_UpdateUser(t *testing.T) { + validBody, err := xml.Marshal(auth.MutableProps{ + Secret: utils.GetStringPtr("secret"), + Role: auth.RoleAdmin, + }) + assert.NoError(t, err) + + invalidUserRoleBody, err := xml.Marshal(auth.MutableProps{ + Secret: utils.GetStringPtr("secret"), + Role: auth.Role("invalid_role"), + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "missing user access key", + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminMissingUserAcess), + }, + }, + { + name: "invalid request body", + input: testInput{ + body: []byte("invalid_request_body"), + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrMalformedXML), + }, + }, + { + name: "invalid user role", + input: testInput{ + body: invalidUserRoleBody, + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminInvalidUserRole), + }, + }, + { + name: "backend returns user not found error", + input: testInput{ + body: validBody, + beErr: auth.ErrNoSuchUser, + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminUserNotFound), + }, + }, + { + name: "backend returns other error", + input: testInput{ + body: validBody, + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + body: validBody, + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iam := &IAMServiceMock{ + UpdateUserAccountFunc: func(access string, props auth.MutableProps) error { + return tt.input.beErr + }, + } + + ctrl := AdminController{ + iam: iam, + } + + testController( + t, + ctrl.UpdateUser, + tt.output.response, + tt.output.err, + ctxInputs{ + body: tt.input.body, + queries: tt.input.queries, + }) + }) + } +} + +func TestAdminController_DeleteUser(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "missing user access key", + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminMissingUserAcess), + }, + }, + { + name: "backend returns other error", + input: testInput{ + beErr: s3err.GetAPIError(s3err.ErrInvalidRequest), + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "successful response", + input: testInput{ + queries: map[string]string{ + "access": "user", + }, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iam := &IAMServiceMock{ + DeleteUserAccountFunc: func(access string) error { + return tt.input.beErr + }, + } + + ctrl := AdminController{ + iam: iam, + } + + testController( + t, + ctrl.DeleteUser, + tt.output.response, + tt.output.err, + ctxInputs{ + queries: tt.input.queries, + }) + }) + } +} + +func TestAdminController_ListUsers(t *testing.T) { + accs := []auth.Account{ + { + Access: "access", + Secret: "secret", + }, + { + Access: "access", + Secret: "secret", + }, + } + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "backend returns error", + input: testInput{ + beRes: []auth.Account{}, + beErr: s3err.GetAPIError(s3err.ErrInternalError), + }, + output: testOutput{ + response: &Response{ + Data: auth.ListUserAccountsResult{ + Accounts: []auth.Account{}, + }, + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrInternalError), + }, + }, + { + name: "successful response", + input: testInput{ + beRes: accs, + }, + output: testOutput{ + response: &Response{ + Data: auth.ListUserAccountsResult{ + Accounts: accs, + }, + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iam := &IAMServiceMock{ + ListUserAccountsFunc: func() ([]auth.Account, error) { + return tt.input.beRes.([]auth.Account), tt.input.beErr + }, + } + + ctrl := AdminController{ + iam: iam, + } + + testController( + t, + ctrl.ListUsers, + tt.output.response, + tt.output.err, + ctxInputs{ + queries: tt.input.queries, + }) + }) + } +} + +func TestAdminController_ChangeBucketOwner(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "fails to get user account", + input: testInput{ + extraMockErr: s3err.GetAPIError(s3err.ErrInternalError), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: errors.New("check user account: "), + }, + }, + { + name: "user not found", + input: testInput{ + extraMockErr: auth.ErrNoSuchUser, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminUserNotFound), + }, + }, + { + name: "backend returns error", + input: testInput{ + beErr: s3err.GetAPIError(s3err.ErrAdminMethodNotSupported), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrAdminMethodNotSupported), + }, + }, + { + name: "successful response", + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iam := &IAMServiceMock{ + GetUserAccountFunc: func(access string) (auth.Account, error) { + return auth.Account{}, tt.input.extraMockErr + }, + } + be := &BackendMock{ + ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error { + return tt.input.beErr + }, + } + + ctrl := AdminController{ + iam: iam, + be: be, + } + + testController( + t, + ctrl.ChangeBucketOwner, + tt.output.response, + tt.output.err, + ctxInputs{}, + ) + }) + } +} + +func TestAdminController_ListBuckets(t *testing.T) { + res := []s3response.Bucket{ + { + Name: "bucket", + Owner: "owner", + }, + } + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "backend returns other error", + input: testInput{ + beRes: []s3response.Bucket{}, + beErr: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListBucketsResult{ + Buckets: []s3response.Bucket{}, + }, + MetaOpts: &MetaOptions{}, + }, + err: s3err.GetAPIError(s3err.ErrNoSuchBucket), + }, + }, + { + name: "successful response", + input: testInput{ + beRes: res, + }, + output: testOutput{ + response: &Response{ + Data: s3response.ListBucketsResult{ + Buckets: res, + }, + MetaOpts: &MetaOptions{}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) { + return tt.input.beRes.([]s3response.Bucket), tt.input.beErr + }, + } + + ctrl := AdminController{ + be: be, + } + + testController( + t, + ctrl.ListBuckets, + tt.output.response, + tt.output.err, + ctxInputs{}, + ) + }) + } +} From 394675a5a8243f04e2b1e514d5a9066557ff1af2 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Wed, 16 Jul 2025 00:14:29 +0400 Subject: [PATCH 18/20] feat: implements unit tests for controller utilities --- metrics/metrics.go | 24 +- s3api/controllers/admin_test.go | 27 + s3api/controllers/base.go | 1890 +------------------------- s3api/controllers/base_test.go | 576 +++++++- s3api/controllers/bucket-put.go | 36 +- s3api/controllers/bucket-put_test.go | 290 +++- s3api/controllers/object-get.go | 4 +- s3api/router.go | 2 +- s3api/server.go | 2 +- s3api/utils/utils.go | 9 + 10 files changed, 949 insertions(+), 1911 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 22cad81e..753b4057 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -41,8 +41,14 @@ type Tag struct { Value string } -// Manager is a manager of metrics plugins -type Manager struct { +// Manager is the interface definition for metrics manager +type Manager interface { + Send(ctx *fiber.Ctx, err error, action string, count int64, status int) + Close() +} + +// manager is a manager of metrics plugins +type manager struct { wg sync.WaitGroup ctx context.Context @@ -59,7 +65,7 @@ type Config struct { } // NewManager initializes metrics plugins and returns a new metrics manager -func NewManager(ctx context.Context, conf Config) (*Manager, error) { +func NewManager(ctx context.Context, conf Config) (Manager, error) { if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 { return nil, nil } @@ -74,7 +80,7 @@ func NewManager(ctx context.Context, conf Config) (*Manager, error) { addDataChan := make(chan datapoint, dataItemCount) - mgr := &Manager{ + mgr := &manager{ addDataChan: addDataChan, ctx: ctx, config: conf, @@ -112,7 +118,7 @@ func NewManager(ctx context.Context, conf Config) (*Manager, error) { return mgr, nil } -func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) { +func (m *manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) { // In case of Authentication failures, url parsing ... if action == "" { action = ActionUndetected @@ -168,12 +174,12 @@ func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, st } // increment increments the key by one -func (m *Manager) increment(key string, tags ...Tag) { +func (m *manager) increment(key string, tags ...Tag) { m.add(key, 1, tags...) } // add adds value to key -func (m *Manager) add(key string, value int64, tags ...Tag) { +func (m *manager) add(key string, value int64, tags ...Tag) { if m.ctx.Err() != nil { return } @@ -192,7 +198,7 @@ func (m *Manager) add(key string, value int64, tags ...Tag) { } // Close closes metrics channels, waits for data to complete, closes all plugins -func (m *Manager) Close() { +func (m *manager) Close() { // drain the datapoint channels close(m.addDataChan) m.wg.Wait() @@ -209,7 +215,7 @@ type publisher interface { Close() } -func (m *Manager) addForwarder(addChan <-chan datapoint) { +func (m *manager) addForwarder(addChan <-chan datapoint) { for data := range addChan { for _, s := range m.publishers { s.Add(data.key, data.value, data.tags...) diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go index 44696200..e38b79fa 100644 --- a/s3api/controllers/admin_test.go +++ b/s3api/controllers/admin_test.go @@ -23,11 +23,38 @@ import ( "github.com/stretchr/testify/assert" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3log" "github.com/versity/versitygw/s3response" ) +func TestNewAdminController(t *testing.T) { + type args struct { + iam auth.IAMService + be backend.Backend + l s3log.AuditLogger + } + tests := []struct { + name string + args args + want AdminController + }{ + { + name: "initialize admin api", + args: args{}, + want: AdminController{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewAdminController(tt.args.iam, tt.args.be, tt.args.l) + assert.Equal(t, got, tt.want) + }) + } +} + func TestAdminController_CreateUser(t *testing.T) { validBody, err := xml.Marshal(auth.Account{ Access: "access", diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 1ccf4e5f..f7880265 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -16,18 +16,10 @@ package controllers import ( "encoding/xml" - "errors" "fmt" - "io" "net/http" - "net/url" "os" - "strconv" - "strings" - "time" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" @@ -37,7 +29,6 @@ import ( "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" - "github.com/versity/versitygw/s3response" ) type S3ApiController struct { @@ -45,7 +36,7 @@ type S3ApiController struct { iam auth.IAMService logger s3log.AuditLogger evSender s3event.S3EventSender - mm *metrics.Manager + mm metrics.Manager debug bool readonly bool } @@ -61,7 +52,7 @@ var ( xmlhdr = []byte(`` + "\n") ) -func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) S3ApiController { +func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, mm metrics.Manager, debug bool, readonly bool) S3ApiController { if debug { debuglogger.SetDebugEnabled() } @@ -82,1869 +73,8 @@ func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { return &Response{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) } -func getint64(i *int64) int64 { - if i == nil { - return 0 - } - return *i -} - -func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acl := ctx.Get("X-Amz-Acl") - grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") - grantRead := ctx.Get("X-Amz-Grant-Read") - grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") - grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - objectOwnership := types.ObjectOwnership( - ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), - ) - // mfa := ctx.Get("X-Amz-Mfa") - // contentMD5 := ctx.Get("Content-MD5") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - - if ctx.Request().URI().QueryArgs().Has("tagging") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - - tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitBucket) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketTaggingAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketTagging(ctx.Context(), bucket, tagging) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketTagging, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - if ctx.Request().URI().QueryArgs().Has("ownershipControls") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var ownershipControls s3response.OwnershipControls - if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil { - if c.debug { - debuglogger.Logf("failed to unmarshal request body: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - rulesCount := len(ownershipControls.Rules) - isValidOwnership := utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) - if rulesCount != 1 || !isValidOwnership { - if c.debug && rulesCount != 1 { - debuglogger.Logf("ownership control rules should be 1, got %v", rulesCount) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketOwnershipControlsAction, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketOwnershipControls, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("versioning") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketVersioningAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - var versioningConf types.VersioningConfiguration - err = xml.Unmarshal(ctx.Body(), &versioningConf) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling versioning configuration: %v", - err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - if versioningConf.Status != types.BucketVersioningStatusEnabled && - versioningConf.Status != types.BucketVersioningStatusSuspended { - if c.debug { - debuglogger.Logf("invalid versioning configuration status: %v", versioningConf.Status) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketVersioning, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("object-lock") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketObjectLockConfigurationAction, - IsBucketPublic: isPublicBucket, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - config, err := auth.ParseBucketLockConfigurationInput(ctx.Body()) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectLockConfiguration(ctx.Context(), bucket, config) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLockConfiguration, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("cors") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketCorsAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketCors, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketCors(ctx.Context(), []byte{}) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketCors, - BucketOwner: parsedAcl.Owner, - }, - ) - } - } - if ctx.Request().URI().QueryArgs().Has("policy") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketPolicyAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }, - ) - } - - err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body()) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketPolicy, - BucketOwner: parsedAcl.Owner, - }) - } - - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP - - if ctx.Request().URI().QueryArgs().Has("acl") { - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - var input *auth.PutBucketAclInput - - ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) - if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if ownership == types.ObjectOwnershipBucketOwnerEnforced { - if c.debug { - debuglogger.Logf("bucket acls are disabled") - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAclNotSupported), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWriteAcp, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.PutBucketAclAction, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if len(ctx.Body()) > 0 { - var accessControlPolicy auth.AccessControlPolicy - err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling access control policy: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedACL), - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = accessControlPolicy.Validate() - if err != nil { - if c.debug { - debuglogger.Logf("invalid access control policy: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if *accessControlPolicy.Owner.ID != parsedAcl.Owner { - if c.debug { - debuglogger.Logf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner) - } - return SendResponse(ctx, s3err.APIError{ - Code: "InvalidArgument", - Description: "Invalid id", - HTTPStatusCode: http.StatusBadRequest, - }, - &MetaOpts{ - Logger: c.logger, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if grants+acl != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrUnexpectedContent), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &accessControlPolicy, - } - } else if acl != "" { - if acl != "private" && acl != "public-read" && acl != "public-read-write" { - if c.debug { - debuglogger.Logf("invalid acl: %q", acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - ACL: types.BucketCannedACL(acl), - } - } else if grants != "" { - input = &auth.PutBucketAclInput{ - Bucket: &bucket, - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - } - } else { - if c.debug { - debuglogger.Logf("none of the bucket acl options has been specified: canned, req headers, req body") - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrMissingSecurityHeader), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam, acct.Role == auth.RoleAdmin) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutBucketAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - if ok := utils.IsValidOwnership(objectOwnership); !ok { - return SendResponse(ctx, s3err.APIError{ - Code: "InvalidArgument", - Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership), - HTTPStatusCode: http.StatusBadRequest, - }, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced { - if c.debug { - debuglogger.Logf("bucket acls are disabled for %v object ownership", objectOwnership) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - if acl != "" && grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - defACL := auth.ACL{ - Owner: acct.Access, - } - - updAcl, err := auth.UpdateACL(&auth.PutBucketAclInput{ - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - AccessControlPolicy: &auth.AccessControlPolicy{ - Owner: &types.Owner{ - ID: &acct.Access, - }}, - ACL: types.BucketCannedACL(acl), - }, defACL, c.iam, acct.Role == auth.RoleAdmin) - if err != nil { - if c.debug { - debuglogger.Logf("failed to update bucket acl: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) - } - - lockHeader := ctx.Get("X-Amz-Bucket-Object-Lock-Enabled") - // CLI provides "True", SDK - "true" - lockEnabled := lockHeader == "True" || lockHeader == "true" - - err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{ - Bucket: &bucket, - ObjectOwnership: objectOwnership, - ObjectLockEnabledForBucket: &lockEnabled, - }, updAcl) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCreateBucket, - BucketOwner: acct.Access, - }) -} - -func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - keyStart := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - versionId := ctx.Query("versionId") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - contentType := ctx.Get("Content-Type") - contentEncoding := ctx.Get("Content-Encoding") - contentDisposition := ctx.Get("Content-Disposition") - contentLanguage := ctx.Get("Content-Language") - cacheControl := ctx.Get("Cache-Control") - expires := ctx.Get("Expires") - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - tagging := ctx.Get("x-amz-tagging") - - // Copy source headers - copySource := ctx.Get("X-Amz-Copy-Source") - if len(copySource) > 0 && copySource[0] == '/' { - copySource = copySource[1:] - } - copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") - copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") - copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") - copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") - copySrcRange := ctx.Get("X-Amz-Copy-Source-Range") - directive := ctx.Get("X-Amz-Metadata-Directive") - - // Permission headers - acl := ctx.Get("X-Amz-Acl") - grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") - grantRead := ctx.Get("X-Amz-Grant-Read") - grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") - grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") - - // Content Length - contentLengthStr := ctx.Get("Content-Length") - if contentLengthStr == "" { - contentLengthStr = "0" - } - // Use decoded content length if available because the - // middleware will decode the chunked transfer encoding - decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") - if decodedLength != "" { - contentLengthStr = decodedLength - } - - // Other headers - bucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner") - storageClass := ctx.Get("X-Amz-Storage-Class") - - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP - - if keyEnd != "" { - keyStart = strings.Join([]string{keyStart, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && keyStart[len(keyStart)-1:] != "/" { - keyStart = keyStart + "/" - } - - if ctx.Request().URI().QueryArgs().Has("tagging") { - tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutBucketTaggingAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectTagging(ctx.Context(), bucket, keyStart, tagging) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionPutObjectTagging, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectTaggingPut, - }) - } - - if ctx.Request().URI().QueryArgs().Has("retention") { - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectRetentionAction, - IsBucketPublic: IsBucketPublic, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - bypass := strings.EqualFold(bypassHdr, "true") - if bypass { - policy, err := c.be.GetBucketPolicy(ctx.Context(), bucket) - if err != nil { - bypass = false - } else { - if err := auth.VerifyBucketPolicy(policy, acct.Access, bucket, keyStart, auth.BypassGovernanceRetentionAction); err != nil { - bypass = false - } - } - } - - retention, err := auth.ParseObjectLockRetentionInput(ctx.Body()) - if err != nil { - if c.debug { - debuglogger.Logf("failed to parse object lock configuration input: %v", err) - } - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.PutObjectRetention(ctx.Context(), bucket, keyStart, versionId, bypass, retention) - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectRetention, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("legal-hold") { - var legalHold types.ObjectLockLegalHold - if err := xml.Unmarshal(ctx.Body(), &legalHold); err != nil { - if c.debug { - debuglogger.Logf("failed to parse request body: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if legalHold.Status != types.ObjectLockLegalHoldStatusOff && legalHold.Status != types.ObjectLockLegalHoldStatusOn { - if c.debug { - debuglogger.Logf("invalid legal hold status: %v", legalHold.Status) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectLegalHoldAction, - IsBucketPublic: IsBucketPublic, - }); err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - err := c.be.PutObjectLegalHold(ctx.Context(), bucket, keyStart, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) - return SendResponse(ctx, err, &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectLegalHold, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("uploadId") && - ctx.Request().URI().QueryArgs().Has("partNumber") && - copySource != "" { - - cs := copySource - copySource, err := url.QueryUnescape(copySource) - if err != nil { - if c.debug { - debuglogger.Logf("error unescaping copy source %q: %v", - cs, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - partNumber := int32(ctx.QueryInt("partNumber", -1)) - if partNumber < 1 || partNumber > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumber) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, - auth.AccessOptions{ - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - resp, err := c.be.UploadPartCopy(ctx.Context(), - &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &keyStart, - CopySource: ©Source, - PartNumber: &partNumber, - UploadId: &uploadId, - ExpectedBucketOwner: &bucketOwner, - CopySourceRange: ©SrcRange, - }) - if err == nil && resp.CopySourceVersionId != "" { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-copy-source-version-id", - Value: resp.CopySourceVersionId, - }, - }) - } - return SendXMLResponse(ctx, resp, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPartCopy, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("uploadId") && - ctx.Request().URI().QueryArgs().Has("partNumber") { - partNumber := int32(ctx.QueryInt("partNumber", -1)) - if partNumber < 1 || partNumber > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumber) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing content length %q: %v", - contentLengthStr, err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) - if err != nil { - if c.debug { - debuglogger.Logf("err parsing checksum headers: %v", err) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var body io.Reader - bodyi := utils.ContextKeyBodyReader.Get(ctx) - if bodyi != nil { - body = bodyi.(io.Reader) - } else { - body = ctx.Request().BodyStream() - } - - res, err := c.be.UploadPart(ctx.Context(), - &s3.UploadPartInput{ - Bucket: &bucket, - Key: &keyStart, - UploadId: &uploadId, - PartNumber: &partNumber, - ContentLength: &contentLength, - Body: body, - ChecksumAlgorithm: algorithm, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), - }) - if err == nil { - headers := []utils.CustomHeader{} - if res.ETag != nil { - headers = append(headers, utils.CustomHeader{ - Key: "ETag", - Value: *res.ETag, - }) - } - switch { - case res.ChecksumCRC32 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - - utils.SetResponseHeaders(ctx, headers) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - Action: metrics.ActionUploadPart, - BucketOwner: parsedAcl.Owner, - }) - } - - if ctx.Request().URI().QueryArgs().Has("acl") { - var input *s3.PutObjectAclInput - - if len(ctx.Body()) > 0 { - if grants+acl != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl)", - grants, acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - var accessControlPolicy auth.AccessControlPolicy - err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling access control policy: %v", - err) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - //TODO: This part will be changed when object acls are implemented - - grants := []types.Grant{} - for _, grt := range accessControlPolicy.AccessControlList.Grants { - grants = append(grants, types.Grant{ - Grantee: &types.Grantee{ - ID: &grt.Grantee.ID, - Type: grt.Grantee.Type, - }, - Permission: types.Permission(grt.Permission), - }) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - ACL: "", - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: accessControlPolicy.Owner, - Grants: grants, - }, - } - } - if acl != "" { - if acl != "private" && acl != "public-read" && acl != "public-read-write" { - if c.debug { - debuglogger.Logf("invalid acl: %q", acl) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - if len(ctx.Body()) > 0 || grants != "" { - if c.debug { - debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", - grants, acl, len(ctx.Body())) - } - return SendResponse(ctx, - s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - }) - } - - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - ACL: types.ObjectCannedACL(acl), - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &bucketOwner}, - }, - } - } - if grants != "" { - input = &s3.PutObjectAclInput{ - Bucket: &bucket, - Key: &keyStart, - GrantFullControl: &grantFullControl, - GrantRead: &grantRead, - GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, - GrantWriteACP: &grantWriteACP, - AccessControlPolicy: &types.AccessControlPolicy{ - Owner: &types.Owner{ID: &bucketOwner}, - }, - ACL: "", - } - } - - err := c.be.PutObjectAcl(ctx.Context(), input) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionPutObjectAcl, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectAclPut, - }) - } - - if copySource != "" { - cs := copySource - copySource, err := url.QueryUnescape(copySource) - if err != nil { - if c.debug { - debuglogger.Logf("error unescaping copy source %q: %v", - cs, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, - auth.AccessOptions{ - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - }) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var mtime *time.Time - var umtime *time.Time - if copySrcModifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcModifSince) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing copy source modified since %q: %v", - copySrcModifSince, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - mtime = &tm - } - if copySrcUnmodifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing copy source unmodified since %q: %v", - copySrcUnmodifSince, err) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidCopySource), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - umtime = &tm - } - - metadata := utils.GetUserMetaData(&ctx.Request().Header) - - if directive != "" && directive != "COPY" && directive != "REPLACE" { - if c.debug { - debuglogger.Logf("invalid metadata directive: %v", directive) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidMetadataDirective), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - metaDirective := types.MetadataDirectiveCopy - if directive == "REPLACE" { - metaDirective = types.MetadataDirectiveReplace - } - - tDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive")) - if tDirective != "" && tDirective != types.TaggingDirectiveCopy && tDirective != types.TaggingDirectiveReplace { - if c.debug { - debuglogger.Logf("invalid tagging direcrive: %v", tDirective) - } - return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidTaggingDirective), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - taggingDirective := types.TaggingDirectiveCopy - if tDirective == types.TaggingDirectiveReplace { - taggingDirective = types.TaggingDirectiveReplace - } - - checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) - err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) - if err != nil { - return SendXMLResponse(ctx, nil, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - - objLock, err := utils.ParsObjectLockHdrs(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.CopyObject(ctx.Context(), - s3response.CopyObjectInput{ - Bucket: &bucket, - Key: &keyStart, - ContentType: &contentType, - ContentDisposition: &contentDisposition, - ContentEncoding: &contentEncoding, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - Tagging: &tagging, - TaggingDirective: taggingDirective, - CopySource: ©Source, - CopySourceIfMatch: ©SrcIfMatch, - CopySourceIfNoneMatch: ©SrcIfNoneMatch, - CopySourceIfModifiedSince: mtime, - CopySourceIfUnmodifiedSince: umtime, - ExpectedBucketOwner: &acct.Access, - Metadata: metadata, - MetadataDirective: metaDirective, - StorageClass: types.StorageClass(storageClass), - ChecksumAlgorithm: checksumAlgorithm, - ObjectLockRetainUntilDate: &objLock.RetainUntilDate, - ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, - ObjectLockMode: objLock.ObjectLockMode, - }) - if err == nil { - hdrs := []utils.CustomHeader{} - if getstring(res.VersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - if getstring(res.CopySourceVersionId) != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-copy-source-version-id", - Value: getstring(res.CopySourceVersionId), - }) - } - utils.SetResponseHeaders(ctx, hdrs) - - return SendXMLResponse(ctx, res.CopyObjectResult, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - ObjectETag: res.CopyObjectResult.ETag, - VersionId: res.VersionId, - EventName: s3event.EventObjectCreatedCopy, - }) - } else { - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionCopyObject, - BucketOwner: parsedAcl.Owner, - }) - } - } - - metadata := utils.GetUserMetaData(&ctx.Request().Header) - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: keyStart, - Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &keyStart}}, true, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - if c.debug { - debuglogger.Logf("error parsing content length %q: %v", - contentLengthStr, err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - objLock, err := utils.ParsObjectLockHdrs(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - }) - } - - var body io.Reader - bodyi := utils.ContextKeyBodyReader.Get(ctx) - if bodyi != nil { - body = bodyi.(io.Reader) - } else { - body = ctx.Request().BodyStream() - } - - res, err := c.be.PutObject(ctx.Context(), - s3response.PutObjectInput{ - Bucket: &bucket, - Key: &keyStart, - ContentLength: &contentLength, - ContentType: &contentType, - ContentEncoding: &contentEncoding, - ContentDisposition: &contentDisposition, - ContentLanguage: &contentLanguage, - CacheControl: &cacheControl, - Expires: &expires, - Metadata: metadata, - Body: body, - Tagging: &tagging, - ObjectLockRetainUntilDate: &objLock.RetainUntilDate, - ObjectLockMode: objLock.ObjectLockMode, - ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, - ChecksumAlgorithm: algorithm, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), - ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - EvSender: c.evSender, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - ObjectSize: contentLength, - EventName: s3event.EventObjectCreatedPut, - }) - } - hdrs := []utils.CustomHeader{ - { - Key: "ETag", - Value: res.ETag, - }, - } - - if res.VersionID != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: res.VersionID, - }) - } - switch { - case res.ChecksumCRC32 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - if res.ChecksumType != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-checksum-type", - Value: string(res.ChecksumType), - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - ContentLength: contentLength, - EvSender: c.evSender, - Action: metrics.ActionPutObject, - BucketOwner: parsedAcl.Owner, - ObjectETag: &res.ETag, - ObjectSize: contentLength, - EventName: s3event.EventObjectCreatedPut, - }) -} - -func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - var dObj s3response.DeleteObjects - - err := xml.Unmarshal(ctx.Body(), &dObj) - if err != nil { - if c.debug { - debuglogger.Logf("error unmarshalling delete objects: %v", err) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.DeleteObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - // The AWS CLI sends 'True', while Go SDK sends 'true' - bypass := strings.EqualFold(bypassHdr, "true") - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.DeleteObjects(ctx.Context(), - &s3.DeleteObjectsInput{ - Bucket: &bucket, - Delete: &types.Delete{ - Objects: dObj.Objects, - }, - }) - return SendXMLResponse(ctx, res, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjects, - ObjectCount: int64(len(dObj.Objects)), - BucketOwner: parsedAcl.Owner, - EvSender: c.evSender, - EventName: s3event.EventObjectRemovedDeleteObjects, - }) -} - -func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - uploadId := ctx.Query("uploadId") - versionId := ctx.Query("versionId") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - bypassHdr := ctx.Get("X-Amz-Bypass-Governance-Retention") - - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - if ctx.Request().URI().QueryArgs().Has("tagging") { - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.DeleteObjectTaggingAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObjectTagging, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.DeleteObjectTagging(ctx.Context(), bucket, key) - return SendResponse(ctx, err, - &MetaOpts{ - Status: http.StatusNoContent, - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObjectTagging, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectTaggingDelete, - }) - } - - if uploadId != "" { - expectedBucketOwner := ctx.Get("X-Amz-Expected-Bucket-Owner") - requestPayer := ctx.Get("X-Amz-Request-Payer") - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.AbortMultipartUploadAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionAbortMultipartUpload, - BucketOwner: parsedAcl.Owner, - }) - } - - err = c.be.AbortMultipartUpload(ctx.Context(), - &s3.AbortMultipartUploadInput{ - UploadId: &uploadId, - Bucket: &bucket, - Key: &key, - ExpectedBucketOwner: &expectedBucketOwner, - RequestPayer: types.RequestPayer(requestPayer), - }) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionAbortMultipartUpload, - BucketOwner: parsedAcl.Owner, - Status: http.StatusNoContent, - }) - } - - //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionWrite, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.DeleteObjectAction, - IsBucketPublic: IsBucketPublic, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - }) - } - - // The AWS CLI sends 'True', while Go SDK sends 'true' - bypass := strings.EqualFold(bypassHdr, "true") - - err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key, VersionId: &versionId}}, bypass, IsBucketPublic, c.be) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.DeleteObject(ctx.Context(), - &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &key, - VersionId: &versionId, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectRemovedDelete, - Status: http.StatusNoContent, - }) - } - - hdrs := []utils.CustomHeader{} - if res.VersionId != nil && *res.VersionId != "" { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: *res.VersionId, - }) - } - if res.DeleteMarker != nil && *res.DeleteMarker { - hdrs = append(hdrs, utils.CustomHeader{ - Key: "x-amz-delete-marker", - Value: "true", - }) - } - - utils.SetResponseHeaders(ctx, hdrs) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - EvSender: c.evSender, - Action: metrics.ActionDeleteObject, - BucketOwner: parsedAcl.Owner, - EventName: s3event.EventObjectRemovedDelete, - Status: http.StatusNoContent, - }) -} - -func (c S3ApiController) HeadBuckets(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - region := utils.ContextKeyRegion.Get(ctx).(string) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Action: auth.ListBucketAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadBucket, - BucketOwner: parsedAcl.Owner, - }) - } - - _, err = c.be.HeadBucket(ctx.Context(), - &s3.HeadBucketInput{ - Bucket: &bucket, - }) - - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "X-Amz-Access-Point-Alias", - Value: "false", - }, - { - Key: "X-Amz-Bucket-Region", - Value: region, - }, - }) - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadBucket, - BucketOwner: parsedAcl.Owner, - }) -} - -const ( - timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" -) - -type MetaOpts struct { - Logger s3log.AuditLogger - EvSender s3event.S3EventSender - MetricsMng *metrics.Manager +// MetaOptions holds the metadata for metrics, audit logs and s3 events +type MetaOptions struct { ContentLength int64 BucketOwner string ObjectSize int64 @@ -1969,7 +99,7 @@ type Response struct { type Services struct { Logger s3log.AuditLogger EventSender s3event.S3EventSender - MetricsManager *metrics.Manager + MetricsManager metrics.Manager } // Controller is the type definition for an s3api controller @@ -2001,7 +131,7 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand // WrapMiddleware executes the given middleware and handles sending the audit logs // and metrics. It also handles the error parsing -func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { +func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.Manager) fiber.Handler { return func(ctx *fiber.Ctx) error { err := handler(ctx) if err != nil { @@ -2094,11 +224,12 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s responseBytes = encodedResp } else { if responseBytes, err = xml.Marshal(response.Data); err != nil { - return err + debuglogger.Logf("Internal Error, %v", err) + return ctx.Status(http.StatusInternalServerError).Send(s3err.GetAPIErrorResponse( + s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) } if len(responseBytes) > 0 { - ctx.Response().Header.Set("Content-Length", fmt.Sprint(len(responseBytes))) ctx.Response().Header.SetContentType(fiber.MIMEApplicationXML) } } @@ -2142,6 +273,9 @@ func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, s res = append(res, xmlhdr...) res = append(res, responseBytes...) + // Set the Content-Length header + ctx.Response().Header.SetContentLength(msglen) + return ctx.Send(res) } diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 5cabf374..ed62aafb 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -16,17 +16,27 @@ package controllers import ( "bytes" + "encoding/xml" + "errors" + "fmt" "net/http" "net/http/httptest" "net/url" "path" + "strings" "testing" "github.com/gofiber/fiber/v2" "github.com/stretchr/testify/assert" + "github.com/valyala/fasthttp" "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/backend" + "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3log" + "github.com/versity/versitygw/s3response" ) var ( @@ -143,8 +153,572 @@ func buildRequest(bucket, object string, body []byte, headers, queries map[strin // set the request headers for key, val := range headers { - req.Header.Add(key, val) + req.Header.Set(key, val) } return req } + +func TestNew(t *testing.T) { + type args struct { + be backend.Backend + iam auth.IAMService + logger s3log.AuditLogger + evs s3event.S3EventSender + mm metrics.Manager + debug bool + readonly bool + } + tests := []struct { + name string + args args + want S3ApiController + }{ + { + name: "debug enabled", + args: args{ + debug: true, + }, + want: S3ApiController{ + debug: true, + }, + }, + { + name: "debug disabled", + args: args{ + debug: false, + }, + want: S3ApiController{ + debug: false, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := New(tt.args.be, tt.args.iam, tt.args.logger, tt.args.evs, tt.args.mm, tt.args.debug, tt.args.readonly) + assert.Equal(t, got, tt.want) + }) + } +} + +func TestS3ApiController_HandleUnmatch(t *testing.T) { + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "return method not allowed", + output: testOutput{ + response: &Response{}, + err: s3err.GetAPIError(s3err.ErrMethodNotAllowed), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := S3ApiController{} + + testController( + t, + ctrl.HandleUnmatch, + tt.output.response, + tt.output.err, + ctxInputs{}) + }) + } +} + +func TestSetResponseHeaders(t *testing.T) { + type args struct { + headers map[string]*string + } + tests := []struct { + name string + args args + expected map[string]string + }{ + { + name: "should not set if map is nil", + args: args{ + headers: nil, + }, + expected: nil, + }, + { + name: "should set some headers", + args: args{ + headers: map[string]*string{ + "x-amz-checksum-algorithm": utils.GetStringPtr("crc32"), + "x-amz-meta-key": utils.GetStringPtr("meta_key"), + "x-amz-mp-size": utils.GetStringPtr(""), + "something": nil, + }, + }, + expected: map[string]string{ + "x-amz-checksum-algorithm": "crc32", + "x-amz-meta-key": "meta_key", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := fiber.New() + ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) + SetResponseHeaders(ctx, tt.args.headers) + if tt.expected != nil { + for key, val := range tt.expected { + v := ctx.Response().Header.Peek(key) + assert.Equal(t, val, string(v)) + } + } + }) + } +} + +// mock the audit logger +type mockAuditLogger struct { +} + +func (m *mockAuditLogger) Log(_ *fiber.Ctx, _ error, _ []byte, _ s3log.LogMeta) {} +func (m *mockAuditLogger) HangUp() error { return nil } +func (m *mockAuditLogger) Shutdown() error { return nil } + +// mock S3 event sender +type mockEvSender struct { +} + +func (m *mockEvSender) SendEvent(_ *fiber.Ctx, _ s3event.EventMeta) {} +func (m *mockEvSender) Close() error { return nil } + +// mock metrics manager + +type mockMetricsManager struct{} + +func (m *mockMetricsManager) Send(_ *fiber.Ctx, _ error, _ string, _ int64, _ int) {} +func (m *mockMetricsManager) Close() {} + +func TestProcessController(t *testing.T) { + payload, err := xml.Marshal(s3response.Bucket{ + Name: "something", + }) + assert.NoError(t, err) + + payloadLen := len(payload) + len(xmlhdr) + + services := &Services{ + Logger: &mockAuditLogger{}, + EventSender: &mockEvSender{}, + MetricsManager: &mockMetricsManager{}, + } + type args struct { + controller Controller + svc *Services + } + type expected struct { + status int + headers map[string]string + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "no services successfull response", + args: args{ + svc: &Services{}, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, nil + }, + }, + expected: expected{ + status: http.StatusOK, + }, + }, + { + name: "handle api error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + }, + }, + expected: expected{ + status: http.StatusBadRequest, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInvalidRequest), "", "", ""), + }, + }, + { + name: "handle custom error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, errors.New("custom error") + }, + }, + expected: expected{ + status: http.StatusInternalServerError, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + { + name: "body parsing fails", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: make(chan int), + }, nil + }, + }, + expected: expected{ + status: http.StatusInternalServerError, + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + { + name: "no data payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{ + ObjectCount: 2, + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + }, + }, + { + name: "should return 204 http status", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + MetaOpts: &MetaOptions{ + Status: http.StatusNoContent, + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusNoContent, + }, + }, + { + name: "already encoded payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: []byte("encoded_data"), + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + body: []byte("encoded_data"), + headers: map[string]string{ + "Content-Length": "12", + }, + }, + }, + { + name: "should set response headers", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Headers: map[string]*string{ + "X-Amz-My-Custom-Header": utils.GetStringPtr("my_value"), + "X-Amz-Meta-My-Meta": utils.GetStringPtr("my_meta"), + }, + }, nil + }, + }, + expected: expected{ + status: http.StatusOK, + headers: map[string]string{ + "X-Amz-My-Custom-Header": "my_value", + "X-Amz-Meta-My-Meta": "my_meta", + }, + }, + }, + { + name: "large paylod: should return internal error", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + type Item struct { + Value string `xml:"value"` + } + + type payload struct { + Items []Item `xml:"item"` + } + + const targetSize = 5 * 1024 * 1024 // 5 MiB + const itemCount = 500 + const valueSize = targetSize / itemCount + + p := payload{ + Items: make([]Item, itemCount), + } + + // Preallocate one shared string of desired size + var sb strings.Builder + sb.Grow(valueSize) + for range valueSize { + sb.WriteByte('A') + } + largeValue := sb.String() + + for i := range p.Items { + p.Items[i].Value = largeValue + } + + return &Response{ + Data: p, + }, nil + }, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + status: http.StatusInternalServerError, + }, + }, + { + name: "not encoded payload", + args: args{ + svc: services, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: s3response.Bucket{ + Name: "something", + }, + }, nil + }, + }, + expected: expected{ + headers: map[string]string{ + "Content-Length": fmt.Sprint(payloadLen), + }, + body: append(xmlhdr, payload...), + status: http.StatusOK, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := fiber.New().AcquireCtx(&fasthttp.RequestCtx{}) + err := ProcessController(ctx, tt.args.controller, metrics.ActionAbortMultipartUpload, tt.args.svc) + assert.NoError(t, err) + + // check the status + assert.Equal(t, tt.expected.status, ctx.Response().StatusCode()) + + // check the response headers to be set + if tt.expected.headers != nil { + for key, val := range tt.expected.headers { + v := ctx.Response().Header.Peek(key) + assert.Equal(t, val, string(v)) + } + } + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + }) + } +} + +func TestProcessHandlers(t *testing.T) { + payload, err := xml.Marshal(s3response.Checksum{ + CRC32: utils.GetStringPtr("crc32"), + }) + assert.NoError(t, err) + + type args struct { + controller Controller + svc *Services + handlers []fiber.Handler + locals map[utils.ContextKey]any + } + type expected struct { + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "should skip the handlers", + args: args{ + locals: map[utils.ContextKey]any{ + utils.ContextKeySkip: true, + }, + }, + }, + { + name: "handler returns error", + args: args{ + handlers: []fiber.Handler{ + func(ctx *fiber.Ctx) error { + return nil + }, + func(ctx *fiber.Ctx) error { + return s3err.GetAPIError(s3err.ErrAccessDenied) + }, + }, + svc: &Services{}, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAccessDenied), "", "", ""), + }, + }, + { + name: "should process the controller", + args: args{ + handlers: []fiber.Handler{ + func(ctx *fiber.Ctx) error { + return nil + }, + func(ctx *fiber.Ctx) error { + return nil + }, + }, + svc: &Services{}, + controller: func(ctx *fiber.Ctx) (*Response, error) { + return &Response{ + Data: s3response.Checksum{ + CRC32: utils.GetStringPtr("crc32"), + }, + }, nil + }, + }, + expected: expected{ + body: append(xmlhdr, payload...), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mdlwr := ProcessHandlers(tt.args.controller, metrics.ActionCreateBucket, tt.args.svc, tt.args.handlers...) + + app := fiber.New() + + app.Post("/:bucket/*", func(ctx *fiber.Ctx) error { + // set the request locals + if tt.args.locals != nil { + for key, val := range tt.args.locals { + key.Set(ctx, val) + } + } + + // call the controller by passing the ctx + err := mdlwr(ctx) + assert.NoError(t, err) + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + + return nil + }) + + app.All("*", func(ctx *fiber.Ctx) error { + return nil + }) + + req := buildRequest("bucket", "object", nil, nil, nil) + + _, err := app.Test(req) + assert.NoError(t, err) + }) + } +} + +func TestWrapMiddleware(t *testing.T) { + type args struct { + handler fiber.Handler + logger s3log.AuditLogger + mm metrics.Manager + } + type expected struct { + body []byte + } + tests := []struct { + name string + args args + expected expected + }{ + { + name: "handler returns no error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return nil + }, + }, + }, + { + name: "handler returns api error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return s3err.GetAPIError(s3err.ErrAclNotSupported) + }, + mm: &mockMetricsManager{}, + logger: &mockAuditLogger{}, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrAclNotSupported), "", "", ""), + }, + }, + { + name: "handler returns custom error", + args: args{ + handler: func(ctx *fiber.Ctx) error { + return errors.New("custom error") + }, + }, + expected: expected{ + body: s3err.GetAPIErrorResponse(s3err.GetAPIError(s3err.ErrInternalError), "", "", ""), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mdlwr := WrapMiddleware(tt.args.handler, tt.args.logger, tt.args.mm) + app := fiber.New() + + app.Post("/:bucket/*", func(ctx *fiber.Ctx) error { + // call the controller by passing the ctx + err := mdlwr(ctx) + assert.NoError(t, err) + + // check the response body + if tt.expected.body != nil { + assert.Equal(t, tt.expected.body, ctx.Response().Body()) + } + + return nil + }) + + app.All("*", func(ctx *fiber.Ctx) error { + return nil + }) + + req := buildRequest("bucket", "object", nil, nil, nil) + + _, err := app.Test(req) + assert.NoError(t, err) + }) + } +} diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index 6080c06b..8a0caf43 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -311,24 +311,7 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP var input *auth.PutBucketAclInput - ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) - if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, err - } - if ownership == types.ObjectOwnershipBucketOwnerEnforced { - debuglogger.Logf("bucket acls are disabled") - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrAclNotSupported) - } - - err = auth.VerifyAccess(ctx.Context(), c.be, + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Readonly: c.readonly, Acl: parsedAcl, @@ -346,6 +329,23 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { }, err } + ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, err + } + if ownership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled") + return &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAclNotSupported) + } + if len(ctx.Body()) > 0 { var accessControlPolicy auth.AccessControlPolicy err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) diff --git a/s3api/controllers/bucket-put_test.go b/s3api/controllers/bucket-put_test.go index 622354ac..239addc7 100644 --- a/s3api/controllers/bucket-put_test.go +++ b/s3api/controllers/bucket-put_test.go @@ -17,6 +17,7 @@ package controllers import ( "context" "encoding/xml" + "errors" "fmt" "net/http" "testing" @@ -810,4 +811,291 @@ func TestS3ApiController_CreateBucket(t *testing.T) { } } -// TODO: add a test for PutBucketAcl +func TestS3ApiController_PutBucketAcl(t *testing.T) { + invalidBody, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{ + Grants: []auth.Grant{ + { + Permission: auth.Permission("invalid_permission"), + }, + }, + }, + }) + assert.NoError(t, err) + + incorrectOwnerBody, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("user"), + }, + AccessControlList: auth.AccessControlList{}, + }) + assert.NoError(t, err) + + validAccessControlPolicy, err := xml.Marshal(auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: utils.GetStringPtr("root"), + }, + AccessControlList: auth.AccessControlList{}, + }) + assert.NoError(t, err) + + tests := []struct { + name string + input testInput + output testOutput + }{ + { + name: "access denied", + input: testInput{ + locals: accessDeniedLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAccessDenied), + }, + }, + { + name: "fails to get bucket ownership", + input: testInput{ + locals: defaultLocals, + extraMockErr: s3err.GetAPIError(s3err.ErrInternalError), + extraMockResp: types.ObjectOwnership(""), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInternalError), + }, + }, + { + name: "acl not supported", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerEnforced, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrAclNotSupported), + }, + }, + { + name: "invalid request body", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: []byte("invalid_body"), + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedACL), + }, + }, + { + name: "invalid access control policy", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: invalidBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMalformedACL), + }, + }, + { + name: "incorrect owner id", + input: testInput{ + locals: defaultLocals, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + body: incorrectOwnerBody, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.APIError{ + Code: "InvalidArgument", + Description: "Invalid id", + HTTPStatusCode: http.StatusBadRequest, + }, + }, + }, + { + name: "both access control policy and grants", + input: testInput{ + body: validAccessControlPolicy, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrUnexpectedContent), + }, + }, + { + name: "access control policy success", + input: testInput{ + body: validAccessControlPolicy, + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "invalid canned acl", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "invalid_acl", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrInvalidRequest), + }, + }, + { + name: "both canned acl and grants", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + "X-Amz-Grant-Read": "grt1", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants), + }, + }, + { + name: "canned acl success", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Acl": "public-read", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, + { + name: "grants update acl fails", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + headers: map[string]string{ + "X-Amz-Grant-Read": "grt1", + }, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: errors.New("accounts does not exist: grt1"), + }, + }, + { + name: "no option provided", + input: testInput{ + extraMockResp: types.ObjectOwnershipBucketOwnerPreferred, + locals: defaultLocals, + }, + output: testOutput{ + response: &Response{ + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + err: s3err.GetAPIError(s3err.ErrMissingSecurityHeader), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + be := &BackendMock{ + PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error { + return tt.input.beErr + }, + GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { + return tt.input.extraMockResp.(types.ObjectOwnership), tt.input.extraMockErr + }, + GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { + return nil, s3err.GetAPIError(s3err.ErrAccessDenied) + }, + } + + ctrl := S3ApiController{ + be: be, + iam: auth.NewIAMServiceSingle( + auth.Account{ + Access: "root", + }), + } + + testController(t, ctrl.PutBucketAcl, tt.output.response, tt.output.err, + ctxInputs{ + locals: tt.input.locals, + body: tt.input.body, + bucket: tt.input.bucket, + headers: tt.input.headers, + }) + }) + } +} diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index b86bffc3..9fd7104c 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -461,7 +461,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { *res.ContentLength) return &Response{ MetaOpts: &MetaOptions{ - ContentLength: getint64(res.ContentLength), + ContentLength: utils.GetInt64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, }, @@ -501,7 +501,7 @@ func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) { "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), }, MetaOpts: &MetaOptions{ - ContentLength: getint64(res.ContentLength), + ContentLength: utils.GetInt64(res.ContentLength), BucketOwner: parsedAcl.Owner, Status: status, }, diff --git a/s3api/router.go b/s3api/router.go index 567ab62d..24356422 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -29,7 +29,7 @@ type S3ApiRouter struct { WithAdmSrv bool } -func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool, region string, root middlewares.RootUserConfig) { +func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm metrics.Manager, debug bool, readonly bool, region string, root middlewares.RootUserConfig) { ctrl := controllers.New(be, iam, logger, evs, mm, debug, readonly) adminServices := &controllers.Services{ Logger: aLogger, diff --git a/s3api/server.go b/s3api/server.go index 857bf810..c25d2650 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -51,7 +51,7 @@ func New( l s3log.AuditLogger, adminLogger s3log.AuditLogger, evs s3event.S3EventSender, - mm *metrics.Manager, + mm metrics.Manager, opts ...Option, ) (*S3ApiServer, error) { server := &S3ApiServer{ diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 5489d9c5..c6f0535c 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -704,3 +704,12 @@ func FormatDatePtrToString(date *time.Time, format string) *string { formatted := date.UTC().Format(format) return &formatted } + +// GetInt64 returns the value of int64 pointer +func GetInt64(n *int64) int64 { + if n == nil { + return 0 + } + + return *n +} From dc16c0448fa8716df6253c4de6f1896ccd37e6aa Mon Sep 17 00:00:00 2001 From: niksis02 Date: Fri, 18 Jul 2025 01:21:50 +0400 Subject: [PATCH 19/20] feat: implements integration tests for the new advanced router --- backend/common.go | 11 +- go.mod | 3 + s3api/controllers/admin.go | 23 +- s3api/controllers/admin_test.go | 2 +- s3api/controllers/base.go | 6 +- s3api/controllers/base_test.go | 15 +- s3api/controllers/object-put.go | 5 +- .../bucket-object-name-validator.go | 26 +-- s3api/middlewares/public-bucket.go | 20 +- s3api/router.go | 64 +++++- s3api/server.go | 14 +- s3err/s3err.go | 6 + tests/integration/group-tests.go | 14 ++ tests/integration/tests.go | 200 +++++++++++++++--- tests/integration/utils.go | 2 +- 15 files changed, 289 insertions(+), 122 deletions(-) diff --git a/backend/common.go b/backend/common.go index 951d4841..7aaa80b0 100644 --- a/backend/common.go +++ b/backend/common.go @@ -22,6 +22,7 @@ import ( "hash" "io" "io/fs" + "math" "net/url" "os" "regexp" @@ -113,7 +114,10 @@ func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error return 0, size, false, nil } - startOffset, err := strconv.ParseInt(bRange[0], 10, 64) + startOffset, err := strconv.ParseInt(bRange[0], 10, strconv.IntSize) + if startOffset > int64(math.MaxInt) || startOffset < int64(math.MinInt) { + return 0, size, false, errInvalidRange + } if err != nil && bRange[0] != "" { return 0, size, false, nil } @@ -128,7 +132,10 @@ func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error return startOffset, size - startOffset, true, nil } - endOffset, err := strconv.ParseInt(bRange[1], 10, 64) + endOffset, err := strconv.ParseInt(bRange[1], 10, strconv.IntSize) + if endOffset > int64(math.MaxInt) { + return 0, size, false, errInvalidRange + } if err != nil { return 0, size, false, nil } diff --git a/go.mod b/go.mod index 6dc68705..bb54d8f5 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/pkg/xattr v0.4.12 github.com/segmentio/kafka-go v0.4.48 github.com/smira/go-statsd v1.3.4 + github.com/stretchr/testify v1.10.0 github.com/urfave/cli/v2 v2.27.7 github.com/valyala/fasthttp v1.64.0 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 @@ -39,6 +40,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.25.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.34.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect github.com/golang-jwt/jwt/v5 v5.2.3 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -57,6 +59,7 @@ require ( golang.org/x/net v0.42.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/s3api/controllers/admin.go b/s3api/controllers/admin.go index c4f5f18e..a85caf0e 100644 --- a/s3api/controllers/admin.go +++ b/s3api/controllers/admin.go @@ -15,13 +15,10 @@ package controllers import ( - "encoding/json" "encoding/xml" - "fmt" "net/http" "strings" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/backend" @@ -149,25 +146,7 @@ func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrAdminUserNotFound) } - acl := auth.ACL{ - Owner: owner, - Grantees: []auth.Grantee{ - { - Permission: auth.PermissionFullControl, - Access: owner, - Type: types.TypeCanonicalUser, - }, - }, - } - - aclParsed, err := json.Marshal(acl) - if err != nil { - return &Response{ - MetaOpts: &MetaOptions{}, - }, fmt.Errorf("failed to marshal the bucket acl: %w", err) - } - - err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed) + err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner) return &Response{ MetaOpts: &MetaOptions{}, }, err diff --git a/s3api/controllers/admin_test.go b/s3api/controllers/admin_test.go index e38b79fa..73ef184b 100644 --- a/s3api/controllers/admin_test.go +++ b/s3api/controllers/admin_test.go @@ -490,7 +490,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) { }, } be := &BackendMock{ - ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error { + ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, owner string) error { return tt.input.beErr }, } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index f7880265..cc46677f 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -69,8 +69,10 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } // Returns MethodNotAllowed for unmatched routes -func (c S3ApiController) HandleUnmatch(ctx *fiber.Ctx) (*Response, error) { - return &Response{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) +func (c S3ApiController) HandleErrorRoute(err error) Controller { + return func(ctx *fiber.Ctx) (*Response, error) { + return &Response{}, err + } } // MetaOptions holds the metadata for metrics, audit logs and s3 events diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index ed62aafb..186726a5 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -201,27 +201,30 @@ func TestNew(t *testing.T) { } } -func TestS3ApiController_HandleUnmatch(t *testing.T) { +func TestS3ApiController_HandleErrorRoute(t *testing.T) { tests := []struct { name string input testInput output testOutput }{ { - name: "return method not allowed", + name: "should return the passed error", + input: testInput{ + extraMockErr: s3err.GetAPIError(s3err.ErrAnonymousCreateMp), + }, output: testOutput{ response: &Response{}, - err: s3err.GetAPIError(s3err.ErrMethodNotAllowed), + err: s3err.GetAPIError(s3err.ErrAnonymousCreateMp), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctrl := S3ApiController{} - + s3Ctrl := S3ApiController{} + ctrl := s3Ctrl.HandleErrorRoute(tt.input.extraMockErr) testController( t, - ctrl.HandleUnmatch, + ctrl, tt.output.response, tt.output.err, ctxInputs{}) diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 124db9a5..28444872 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -15,6 +15,7 @@ package controllers import ( + "bytes" "encoding/xml" "fmt" "io" @@ -266,7 +267,7 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { if bodyi != nil { body = bodyi.(io.Reader) } else { - body = ctx.Request().BodyStream() + body = bytes.NewReader([]byte{}) } res, err := c.be.UploadPart(ctx.Context(), @@ -690,7 +691,7 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { if bodyi != nil { body = bodyi.(io.Reader) } else { - body = ctx.Request().BodyStream() + body = bytes.NewReader([]byte{}) } res, err := c.be.PutObject(ctx.Context(), diff --git a/s3api/middlewares/bucket-object-name-validator.go b/s3api/middlewares/bucket-object-name-validator.go index 1abd8a05..f099afd8 100644 --- a/s3api/middlewares/bucket-object-name-validator.go +++ b/s3api/middlewares/bucket-object-name-validator.go @@ -15,44 +15,28 @@ package middlewares import ( - "net/http" - "github.com/gofiber/fiber/v2" - "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3log" ) // BucketObjectNameValidator extracts and validates // the bucket and object names from the request URI. -func BucketObjectNameValidator(l s3log.AuditLogger, mm *metrics.Manager) fiber.Handler { +func BucketObjectNameValidator() fiber.Handler { return func(ctx *fiber.Ctx) error { - // skip the check for admin apis - if ctx.Method() == http.MethodPatch { - return ctx.Next() - } - - path := ctx.Path() - // skip the check if the operation isn't bucket/object scoped - // e.g ListBuckets - if path == "/" { - return ctx.Next() - } - - bucket, object := parsePath(path) + bucket, object := parsePath(ctx.Path()) // check if the provided bucket name is valid if !utils.IsValidBucketName(bucket) { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidBucketName), l, mm) + return s3err.GetAPIError(s3err.ErrInvalidBucketName) } // check if the provided object name is valid // skip for empty objects: e.g bucket operations: HeadBucket... if object != "" && !utils.IsObjectNameValid(object) { - return sendResponse(ctx, s3err.GetAPIError(s3err.ErrBadRequest), l, mm) + return s3err.GetAPIError(s3err.ErrBadRequest) } - return ctx.Next() + return nil } } diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index 30c7aae9..52383083 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -76,26 +76,10 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm if err != nil { return err } + } else { + utils.ContextKeyBodyReader.Set(ctx, ctx.Request().BodyStream()) } - } - if utils.IsBigDataAction(ctx) { - payloadType := ctx.Get("X-Amz-Content-Sha256") - if utils.IsUnsignedStreamingPayload(payloadType) { - checksumType, err := utils.ExtractChecksumType(ctx) - if err != nil { - return err - } - - wrapBodyReader(ctx, func(r io.Reader) io.Reader { - var cr io.Reader - cr, err = utils.NewUnsignedChunkReader(r, checksumType) - return cr - }) - if err != nil { - return err - } - } } utils.ContextKeyPublicBucket.Set(ctx, true) diff --git a/s3api/router.go b/s3api/router.go index 24356422..c0694b3d 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -21,6 +21,7 @@ import ( "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/controllers" "github.com/versity/versitygw/s3api/middlewares" + "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" ) @@ -109,6 +110,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketTagging, metrics.ActionPutBucketTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -121,6 +123,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketOwnershipControls, metrics.ActionPutBucketOwnershipControls, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketOwnershipControls, auth.PutBucketOwnershipControlsAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -133,6 +136,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketVersioning, metrics.ActionPutBucketVersioning, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketVersioning, auth.PutBucketVersioningAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -145,6 +149,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObjectLockConfiguration, metrics.ActionPutObjectLockConfiguration, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectLockConfiguration, auth.PutBucketObjectLockConfigurationAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -157,6 +162,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketCors, metrics.ActionPutBucketCors, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketCors, auth.PutBucketCorsAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -169,6 +175,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketPolicy, metrics.ActionPutBucketPolicy, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketPolicy, auth.PutBucketPolicyAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -181,6 +188,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutBucketAcl, metrics.ActionPutBucketAcl, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutBucketAcl, auth.PutBucketAclAction, auth.PermissionWriteAcp), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -192,6 +200,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.CreateBucket, metrics.ActionCreateBucket, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCreateBucket, auth.CreateBucketAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -204,6 +213,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.HeadBucket, metrics.ActionHeadBucket, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionHeadBucket, auth.ListBucketAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -218,6 +228,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteBucketTagging, metrics.ActionDeleteBucketTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketTagging, auth.PutBucketTaggingAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -230,6 +241,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteBucketOwnershipControls, metrics.ActionDeleteBucketOwnershipControls, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketOwnershipControls, auth.PutBucketOwnershipControlsAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -242,6 +254,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteBucketPolicy, metrics.ActionDeleteBucketPolicy, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketPolicy, auth.PutBucketPolicyAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -254,6 +267,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteBucketCors, metrics.ActionDeleteBucketCors, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucketCors, auth.PutBucketCorsAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -265,6 +279,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteBucket, metrics.ActionDeleteBucket, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteBucket, auth.DeleteBucketAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -279,6 +294,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketTagging, metrics.ActionGetBucketTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketTagging, auth.GetBucketTaggingAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -291,6 +307,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketOwnershipControls, metrics.ActionGetBucketOwnershipControls, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketOwnershipControls, auth.GetBucketOwnershipControlsAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -303,6 +320,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketVersioning, metrics.ActionGetBucketVersioning, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketVersioning, auth.GetBucketVersioningAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -315,6 +333,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketPolicy, metrics.ActionGetBucketPolicy, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketPolicy, auth.GetBucketPolicyAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -327,6 +346,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketCors, metrics.ActionGetBucketCors, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketCors, auth.GetBucketCorsAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -339,6 +359,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectLockConfiguration, metrics.ActionGetObjectLockConfiguration, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectLockConfiguration, auth.GetBucketObjectLockConfigurationAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -351,6 +372,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetBucketAcl, metrics.ActionGetBucketAcl, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetBucketAcl, auth.GetBucketAclAction, auth.PermissionReadAcp), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -363,6 +385,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.ListMultipartUploads, metrics.ActionListMultipartUploads, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListMultipartUploads, auth.ListBucketMultipartUploadsAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -375,6 +398,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.ListObjectVersions, metrics.ActionListObjectVersions, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjectVersions, auth.ListBucketVersionsAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -387,6 +411,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.ListObjectsV2, metrics.ActionListObjectsV2, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjectsV2, auth.ListBucketAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -398,6 +423,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.ListObjects, metrics.ActionListObjects, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListObjects, auth.ListBucketAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -412,6 +438,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteObjects, metrics.ActionDeleteObjects, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObjects, auth.DeleteObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -425,6 +452,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.HeadObject, metrics.ActionHeadObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionHeadObject, auth.GetObjectAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -439,6 +467,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectTagging, metrics.ActionGetObjectTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectTagging, auth.GetObjectTaggingAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -451,6 +480,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectRetention, metrics.ActionGetObjectRetention, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectRetention, auth.GetObjectRetentionAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -463,6 +493,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectLegalHold, metrics.ActionGetObjectLegalHold, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectLegalHold, auth.GetObjectLegalHoldAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -475,6 +506,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectAcl, metrics.ActionGetObjectAcl, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectAcl, auth.GetObjectAclAction, auth.PermissionReadAcp), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -487,6 +519,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObjectAttributes, metrics.ActionGetObjectAttributes, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObjectAttributes, auth.GetObjectAttributesAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -499,6 +532,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.ListParts, metrics.ActionListParts, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionListParts, auth.ListMultipartUploadPartsAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -510,6 +544,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.GetObject, metrics.ActionGetObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionGetObject, auth.GetObjectAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -524,6 +559,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteObjectTagging, metrics.ActionDeleteObjectTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObjectTagging, auth.DeleteObjectTaggingAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -536,6 +572,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.AbortMultipartUpload, metrics.ActionAbortMultipartUpload, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionAbortMultipartUpload, auth.AbortMultipartUploadAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -547,6 +584,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.DeleteObject, metrics.ActionDeleteObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionDeleteObject, auth.DeleteObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -560,6 +598,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.RestoreObject, metrics.ActionRestoreObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionRestoreObject, auth.RestoreObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -573,6 +612,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.SelectObjectContent, metrics.ActionSelectObjectContent, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionSelectObjectContent, auth.GetObjectAction, auth.PermissionRead), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -585,6 +625,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.CompleteMultipartUpload, metrics.ActionCompleteMultipartUpload, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCompleteMultipartUpload, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -597,6 +638,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.CreateMultipartUpload, metrics.ActionCreateMultipartUpload, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCreateMultipartUpload, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -611,6 +653,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObjectTagging, metrics.ActionPutObjectTagging, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectTagging, auth.PutObjectTaggingAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -623,6 +666,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObjectRetention, metrics.ActionPutObjectRetention, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectRetention, auth.PutObjectRetentionAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -635,6 +679,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObjectLegalHold, metrics.ActionPutObjectLegalHold, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectLegalHold, auth.PutObjectLegalHoldAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -647,6 +692,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObjectAcl, metrics.ActionPutObjectAcl, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObjectAcl, auth.PutObjectAclAction, auth.PermissionWriteAcp), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -660,6 +706,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.UploadPartCopy, metrics.ActionUploadPartCopy, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionUploadPartCopy, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -672,18 +719,32 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.UploadPart, metrics.ActionUploadPart, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionUploadPart, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), middlewares.VerifyMD5Body(), middlewares.ParseAcl(be), )) + + // return error if partNumber is used without uploadId + objectRouter.Put("", + middlewares.MatchQueryArgs("partNumber"), + controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMissingUploadId)), metrics.ActionUndetected, services)) + + // return 'MethodNotAllowed' if uploadId is provided without partNumber + // before the router reaches to 'PutObject' + objectRouter.Put("", + middlewares.MatchQueryArgs("uploadId"), + controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMethodNotAllowed)), metrics.ActionUndetected, services)) + objectRouter.Put("", middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessHandlers( ctrl.CopyObject, metrics.ActionCopyObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionCopyObject, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -695,6 +756,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ ctrl.PutObject, metrics.ActionPutObject, services, + middlewares.BucketObjectNameValidator(), middlewares.AuthorizePublicBucketAccess(be, metrics.ActionPutObject, auth.PutObjectAction, auth.PermissionWrite), middlewares.VerifyPresignedV4Signature(root, iam, region, debug), middlewares.VerifyV4Signature(root, iam, region, debug), @@ -703,5 +765,5 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ )) // Return MethodNotAllowed for all the unmatched routes - app.All("*", controllers.ProcessHandlers(ctrl.HandleUnmatch, metrics.ActionUndetected, services)) + app.All("*", controllers.ProcessHandlers(ctrl.HandleErrorRoute(s3err.GetAPIError(s3err.ErrMethodNotAllowed)), metrics.ActionUndetected, services)) } diff --git a/s3api/server.go b/s3api/server.go index c25d2650..8030b476 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -95,19 +95,7 @@ func New( app.Use(middlewares.DebugLogger()) } - // initialize the bucket/object name validator - app.Use(middlewares.BucketObjectNameValidator(l, mm)) - - // Public buckets access checker - app.Use(middlewares.AuthorizePublicBucketAccess(be, l, mm)) - - // Authentication middlewares - app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug)) - app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug)) - app.Use(middlewares.VerifyMD5Body(l)) - app.Use(middlewares.AclParser(be, l, server.readonly)) - - server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly) + server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly, region, root) return server, nil } diff --git a/s3err/s3err.go b/s3err/s3err.go index 59aca425..4dd1a374 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -168,6 +168,7 @@ const ( ErrInvalidChecksumHeader ErrTrailerHeaderNotSupported ErrBadRequest + ErrMissingUploadId // Non-AWS errors ErrExistingObjectIsDirectory @@ -732,6 +733,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Bad Request", HTTPStatusCode: http.StatusBadRequest, }, + ErrMissingUploadId: { + Code: "InvalidArgument", + Description: "This operation does not accept partNumber without uploadId", + HTTPStatusCode: http.StatusBadRequest, + }, // non aws errors ErrExistingObjectIsDirectory: { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index a3a15f50..e2d7ac7b 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -637,6 +637,7 @@ func TestFullFlow(s *S3Conf) { TestGetObjectLegalHold(s) TestWORMProtection(s) TestAccessControl(s) + TestRouter(s) // FIXME: The tests should pass for azure as well // but this issue should be fixed with https://github.com/versity/versitygw/issues/1336 if !s.azureTests { @@ -652,6 +653,7 @@ func TestPosix(s *S3Conf) { PutObject_overwrite_file_obj(s) PutObject_overwrite_file_obj_with_nested_obj(s) PutObject_dir_obj_with_data(s) + PutObject_with_slashes(s) CreateMultipartUpload_dir_obj(s) PutObject_name_too_long(s) HeadObject_name_too_long(s) @@ -749,6 +751,7 @@ func TestScoutfs(s *S3Conf) { PutObject_overwrite_file_obj(s) PutObject_overwrite_file_obj_with_nested_obj(s) PutObject_dir_obj_with_data(s) + PutObject_with_slashes(s) CreateMultipartUpload_dir_obj(s) PutObject_name_too_long(s) HeadObject_name_too_long(s) @@ -872,6 +875,12 @@ func TestVersioningDisabled(s *S3Conf) { VersioningDisabled_PutBucketVersioning_not_configured(s) } +func TestRouter(s *S3Conf) { + RouterPutPartNumberWithoutUploadId(s) + RouterPostRoot(s) + RouterPostObjectWithoutQuery(s) +} + type IntTests map[string]func(s *S3Conf) error func GetIntTests() IntTests { @@ -1274,6 +1283,7 @@ func GetIntTests() IntTests { "PutObject_overwrite_file_obj": PutObject_overwrite_file_obj, "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, + "PutObject_with_slashes": PutObject_with_slashes, "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, "IAM_user_access_denied": IAM_user_access_denied, "IAM_userplus_access_denied": IAM_userplus_access_denied, @@ -1355,5 +1365,9 @@ func GetIntTests() IntTests { "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, "Versioning_concurrent_upload_object": Versioning_concurrent_upload_object, + "RouterPutPartNumberWithoutUploadId": RouterPutPartNumberWithoutUploadId, + "RouterPostRoot": RouterPostRoot, + "RouterPostObjectWithoutQuery": RouterPostObjectWithoutQuery, + "RouterPUTObjectOnlyUploadId": RouterPUTObjectOnlyUploadId, } } diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 7872b5d6..65ac11f9 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -63,7 +63,7 @@ func Authentication_invalid_auth_header(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMissingFields)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMissingFields)); err != nil { return err } @@ -89,7 +89,7 @@ func Authentication_unsupported_signature_version(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported)); err != nil { return err } @@ -116,7 +116,7 @@ func Authentication_malformed_credentials(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { return err } @@ -143,7 +143,7 @@ func Authentication_malformed_credentials_invalid_parts(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { return err } @@ -170,7 +170,7 @@ func Authentication_credentials_terminated_string(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil { return err } @@ -193,7 +193,7 @@ func Authentication_credentials_incorrect_service(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil { return err } @@ -227,7 +227,7 @@ func Authentication_credentials_incorrect_region(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, apiErr); err != nil { + if err := checkHTTPResponseApiErr(resp, apiErr); err != nil { return err } @@ -254,7 +254,7 @@ func Authentication_credentials_invalid_date(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { return err } @@ -355,7 +355,7 @@ func Authentication_credentials_non_existing_access_key(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil { return err } @@ -382,7 +382,7 @@ func Authentication_invalid_signed_headers(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { return err } @@ -406,7 +406,7 @@ func Authentication_missing_date_header(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMissingDateHeader)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMissingDateHeader)); err != nil { return err } @@ -430,7 +430,7 @@ func Authentication_invalid_date_header(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMalformedDate)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMalformedDate)); err != nil { return err } @@ -454,7 +454,7 @@ func Authentication_date_mismatch(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { return err } @@ -478,7 +478,7 @@ func Authentication_incorrect_payload_hash(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)); err != nil { return err } @@ -503,7 +503,7 @@ func Authentication_incorrect_md5(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidDigest)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidDigest)); err != nil { return err } @@ -528,7 +528,7 @@ func Authentication_signature_error_incorrect_secret_key(s *S3Conf) error { return err } defer resp.Body.Close() - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil { return err } @@ -558,7 +558,7 @@ func PresignedAuth_unsupported_algorithm(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQuerySignatureAlgo)); err != nil { return err } @@ -595,7 +595,7 @@ func PresignedAuth_missing_credentials_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { return err } @@ -632,7 +632,7 @@ func PresignedAuth_malformed_creds_invalid_parts(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil { return err } @@ -665,7 +665,7 @@ func PresignedAuth_creds_invalid_terminator(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)); err != nil { return err } @@ -698,7 +698,7 @@ func PresignedAuth_creds_incorrect_service(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureIncorrService)); err != nil { return err } @@ -736,7 +736,7 @@ func PresignedAuth_creds_incorrect_region(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.APIError{ + if err := checkHTTPResponseApiErr(resp, s3err.APIError{ Code: "SignatureDoesNotMatch", Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", cfg.awsRegion), HTTPStatusCode: http.StatusForbidden, @@ -773,7 +773,7 @@ func PresignedAuth_creds_invalid_date(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { return err } @@ -806,7 +806,7 @@ func PresignedAuth_non_existing_access_key_id(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)); err != nil { return err } @@ -843,7 +843,7 @@ func PresignedAuth_missing_date_query(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { return err } @@ -876,7 +876,7 @@ func PresignedAuth_dates_mismatch(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)); err != nil { return err } @@ -913,7 +913,7 @@ func PresignedAuth_missing_signed_headers_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { return err } @@ -950,7 +950,7 @@ func PresignedAuth_missing_expiration_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil { return err } @@ -987,7 +987,7 @@ func PresignedAuth_invalid_expiration_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMalformedExpires)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMalformedExpires)); err != nil { return err } @@ -1024,7 +1024,7 @@ func PresignedAuth_negative_expiration_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrNegativeExpires)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrNegativeExpires)); err != nil { return err } @@ -1061,7 +1061,7 @@ func PresignedAuth_exceeding_expiration_query_param(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrMaximumExpires)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMaximumExpires)); err != nil { return err } @@ -1105,7 +1105,7 @@ func PresignedAuth_expired_request(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrExpiredPresignRequest)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrExpiredPresignRequest)); err != nil { return err } @@ -1137,7 +1137,7 @@ func PresignedAuth_incorrect_secret_key(s *S3Conf) error { return err } - if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil { + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)); err != nil { return err } @@ -17944,6 +17944,47 @@ func PutObject_dir_obj_with_data(s *S3Conf) error { }) } +func PutObject_with_slashes(s *S3Conf) error { + testName := "PutObject_with_slashes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objs, err := putObjects(s3client, []string{ + "/obj", "foo//bar", "/foo/baz/bar", "////////bar", "foo//////quxx", + }, bucket) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + // it's en expected bahvior in posix to normalize the object pahts, + // by removing multiple slashes + normalizedObjs := []string{ + "bar", + "foo/bar", + "foo/baz/bar", + "foo/quxx", + "obj", + } + + for i := range objs { + objs[i].Key = &normalizedObjs[i] + } + + if !compareObjects(objs, res.Contents) { + return fmt.Errorf("expected the objects to be %vß, instead got %v", + objStrings(objs), objStrings(res.Contents)) + } + + return nil + }) +} + func CreateMultipartUpload_dir_obj(s *S3Conf) error { testName := "CreateMultipartUpload_dir_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -20827,3 +20868,96 @@ func Versioning_concurrent_upload_object(s *S3Conf) error { return nil }, withVersioning(types.BucketVersioningStatusEnabled)) } + +// router tests +func RouterPutPartNumberWithoutUploadId(s *S3Conf) error { + testName := "RouterPutPartNumberWithoutUploadId" + return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error { + req, err := http.NewRequest(http.MethodPut, s.endpoint+"/bucket/object", nil) + if err != nil { + return err + } + + query := req.URL.Query() + query.Add("partNumber", "1") + req.URL.RawQuery = query.Encode() + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMissingUploadId)); err != nil { + return err + } + + return nil + }) +} + +func RouterPostRoot(s *S3Conf) error { + testName := "RouterPostRoot" + return actionHandlerNoSetup(s, testName, func(s3client *s3.Client, bucket string) error { + req, err := http.NewRequest(http.MethodPost, s.endpoint+"/", nil) + if err != nil { + return err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil + }) +} + +func RouterPostObjectWithoutQuery(s *S3Conf) error { + testName := "RouterPostObjectWithoutQuery" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + req, err := http.NewRequest(http.MethodPost, s.endpoint+"/bucket/object", nil) + if err != nil { + return err + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil + }) +} + +func RouterPUTObjectOnlyUploadId(s *S3Conf) error { + testName := "RouterPUTObjectOnlyUploadId" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + req, err := http.NewRequest(http.MethodPut, s.endpoint+"/bucket/object", nil) + if err != nil { + return err + } + + query := req.URL.Query() + query.Add("uploadId", "my-upload-id") + req.URL.RawQuery = query.Encode() + + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + + if err := checkHTTPResponseApiErr(resp, s3err.GetAPIError(s3err.ErrMethodNotAllowed)); err != nil { + return err + } + + return nil + }) +} diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 8edcb7bc..1ff2a1aa 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -334,7 +334,7 @@ func createSignedReq(method, endpoint, path, access, secret, service, region str return req, nil } -func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error { +func checkHTTPResponseApiErr(resp *http.Response, apiErr s3err.APIError) error { body, err := io.ReadAll(resp.Body) if err != nil { return err From e74d2c0d197e77a7595b673b433ca53d5380a190 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 21 Jul 2025 18:58:39 +0400 Subject: [PATCH 20/20] fix: fixes the invalid x-amz-mp-object-size header error in CompleteMultipartUpload. Fixes #1398 The `x-amz-mp-object-size` request header can have two erroneous states: an invalid value or a negative integer. AWS returns different error descriptions for each case. This PR fixes the error description for the invalid header value case. The invalid case can't be integration tested as SDK expects `int64` as the header value. --- s3api/controllers/base.go | 7 ++++++- s3api/controllers/bucket-delete.go | 1 + s3api/controllers/bucket-delete_test.go | 2 ++ s3api/controllers/bucket-get.go | 4 +++- s3api/controllers/bucket-list.go | 6 +++--- s3api/controllers/bucket-put.go | 12 ++++++------ s3api/controllers/object-delete.go | 14 +++++++------- s3api/controllers/object-head.go | 2 +- s3api/controllers/object-post.go | 21 ++++++++++----------- s3api/controllers/object-post_test.go | 4 ++-- s3api/controllers/object-put.go | 12 ++++++------ s3err/s3err.go | 10 +++++++++- tests/integration/tests.go | 2 +- 13 files changed, 57 insertions(+), 40 deletions(-) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index cc46677f..eff9cb3d 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -42,10 +42,15 @@ type S3ApiController struct { } const ( + // time constants iso8601Format = "20060102T150405Z" iso8601TimeFormatExtended = "Mon Jan _2 15:04:05 2006" timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" - maxXMLBodyLen = 4 * 1024 * 1024 + + maxXMLBodyLen = 4 * 1024 * 1024 + minPartNumber = 1 + maxPartNumber = 10000 + defaultMaxBuckets = int32(10000) ) var ( diff --git a/s3api/controllers/bucket-delete.go b/s3api/controllers/bucket-delete.go index 00a169dd..22ef9286 100644 --- a/s3api/controllers/bucket-delete.go +++ b/s3api/controllers/bucket-delete.go @@ -153,6 +153,7 @@ func (c S3ApiController) DeleteBucketCors(ctx *fiber.Ctx) (*Response, error) { return &Response{ MetaOpts: &MetaOptions{ BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, }, }, err } diff --git a/s3api/controllers/bucket-delete_test.go b/s3api/controllers/bucket-delete_test.go index 19eb95da..50ae11d9 100644 --- a/s3api/controllers/bucket-delete_test.go +++ b/s3api/controllers/bucket-delete_test.go @@ -286,6 +286,7 @@ func TestS3ApiController_DeleteBucketCors(t *testing.T) { response: &Response{ MetaOpts: &MetaOptions{ BucketOwner: "root", + Status: http.StatusNoContent, }, }, err: s3err.GetAPIError(s3err.ErrAdminMethodNotSupported), @@ -300,6 +301,7 @@ func TestS3ApiController_DeleteBucketCors(t *testing.T) { response: &Response{ MetaOpts: &MetaOptions{ BucketOwner: "root", + Status: http.StatusNoContent, }, }, }, diff --git a/s3api/controllers/bucket-get.go b/s3api/controllers/bucket-get.go index cc60d267..f4da5c34 100644 --- a/s3api/controllers/bucket-get.go +++ b/s3api/controllers/bucket-get.go @@ -61,7 +61,9 @@ func (c S3ApiController) GetBucketTagging(ctx *fiber.Ctx) (*Response, error) { }, err } resp := s3response.Tagging{ - TagSet: s3response.TagSet{Tags: []s3response.Tag{}}, + TagSet: s3response.TagSet{ + Tags: make([]s3response.Tag, 0, len(tags)), + }, } for key, val := range tags { diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go index f990dd1e..5cae8939 100644 --- a/s3api/controllers/bucket-list.go +++ b/s3api/controllers/bucket-list.go @@ -31,10 +31,10 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { maxBucketsStr := ctx.Query("max-buckets") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - var maxBuckets int32 = 10000 + maxBuckets := defaultMaxBuckets if maxBucketsStr != "" { maxBucketsParsed, err := strconv.ParseInt(maxBucketsStr, 10, 32) - if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > 10000 { + if err != nil || maxBucketsParsed < 0 || maxBucketsParsed > int64(defaultMaxBuckets) { debuglogger.Logf("error parsing max-buckets %q: %v", maxBucketsStr, err) return &Response{ MetaOpts: &MetaOptions{}, @@ -47,7 +47,7 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) (*Response, error) { s3response.ListBucketsInput{ Owner: acct.Access, IsAdmin: acct.Role == auth.RoleAdmin, - MaxBuckets: int32(maxBuckets), + MaxBuckets: maxBuckets, ContinuationToken: cToken, Prefix: prefix, }) diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go index 8a0caf43..7fbd0ea3 100644 --- a/s3api/controllers/bucket-put.go +++ b/s3api/controllers/bucket-put.go @@ -301,14 +301,14 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") grantRead := ctx.Get("X-Amz-Grant-Read") grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") + grantWrite := ctx.Get("X-Amz-Grant-Write") grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") // context locals parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + grants := grantFullControl + grantRead + grantReadACP + grantWrite + grantWriteACP var input *auth.PutBucketAclInput err := auth.VerifyAccess(ctx.Context(), c.be, @@ -424,7 +424,7 @@ func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { GrantFullControl: &grantFullControl, GrantRead: &grantRead, GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, + GrantWrite: &grantWrite, GrantWriteACP: &grantWriteACP, } } else { @@ -459,11 +459,11 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") grantRead := ctx.Get("X-Amz-Grant-Read") grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") + grantWrite := ctx.Get("X-Amz-Grant-Write") grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") lockEnabled := strings.EqualFold(ctx.Get("X-Amz-Bucket-Object-Lock-Enabled"), "true") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + grants := grantFullControl + grantRead + grantReadACP + grantWrite + grantWriteACP objectOwnership := types.ObjectOwnership( ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), ) @@ -518,7 +518,7 @@ func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { GrantFullControl: &grantFullControl, GrantRead: &grantRead, GrantReadACP: &grantReadACP, - GrantWrite: &granWrite, + GrantWrite: &grantWrite, GrantWriteACP: &grantWriteACP, AccessControlPolicy: &auth.AccessControlPolicy{ Owner: &types.Owner{ diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go index 93721315..76dfb167 100644 --- a/s3api/controllers/object-delete.go +++ b/s3api/controllers/object-delete.go @@ -32,7 +32,7 @@ func (c S3ApiController) DeleteObjectTagging(ctx *fiber.Ctx) (*Response, error) key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) err := auth.VerifyAccess(ctx.Context(), c.be, @@ -45,7 +45,7 @@ func (c S3ApiController) DeleteObjectTagging(ctx *fiber.Ctx) (*Response, error) Bucket: bucket, Object: key, Action: auth.DeleteObjectTaggingAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -71,7 +71,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) uploadId := ctx.Query("uploadId") acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) err := auth.VerifyAccess(ctx.Context(), c.be, @@ -84,7 +84,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) Bucket: bucket, Object: key, Action: auth.AbortMultipartUploadAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -116,7 +116,7 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) //TODO: check s3:DeleteObjectVersion policy in case a use tries to delete a version of an object @@ -131,7 +131,7 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { Bucket: bucket, Object: key, Action: auth.DeleteObjectAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -152,7 +152,7 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { }, }, bypass, - IsBucketPublic, + isBucketPublic, c.be, ) if err != nil { diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go index 5b0aa304..57c749b0 100644 --- a/s3api/controllers/object-head.go +++ b/s3api/controllers/object-head.go @@ -63,7 +63,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { var partNumber *int32 if ctx.Request().URI().QueryArgs().Has("partNumber") { - if partNumberQuery < 1 || partNumberQuery > 10000 { + if partNumberQuery < minPartNumber || partNumberQuery > maxPartNumber { debuglogger.Logf("invalid part number: %d", partNumberQuery) return &Response{ MetaOpts: &MetaOptions{ diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 658320a4..35fece82 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -36,7 +36,7 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) err := auth.VerifyAccess(ctx.Context(), c.be, @@ -49,7 +49,7 @@ func (c S3ApiController) RestoreObject(ctx *fiber.Ctx) (*Response, error) { Bucket: bucket, Object: key, Action: auth.RestoreObjectAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -87,7 +87,7 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) err := auth.VerifyAccess(ctx.Context(), c.be, @@ -100,7 +100,7 @@ func (c S3ApiController) SelectObjectContent(ctx *fiber.Ctx) (*Response, error) Bucket: bucket, Object: key, Action: auth.GetObjectAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -237,7 +237,7 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) err := auth.VerifyAccess(ctx.Context(), c.be, @@ -250,7 +250,7 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err Bucket: bucket, Object: key, Action: auth.PutObjectAction, - IsBucketPublic: IsBucketPublic, + IsBucketPublic: isBucketPublic, }) if err != nil { return &Response{ @@ -283,23 +283,22 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err var mpuObjectSize *int64 if mpuObjSizeHdr != "" { val, err := strconv.ParseInt(mpuObjSizeHdr, 10, 64) - //TODO: Not sure if invalid request should be returned if err != nil { - debuglogger.Logf("invalid value for 'x-amz-mp-objects-size' header: %v", err) + debuglogger.Logf("invalid value for 'x-amz-mp-object-size' header: %v", err) return &Response{ MetaOpts: &MetaOptions{ BucketOwner: parsedAcl.Owner, }, - }, s3err.GetAPIError(s3err.ErrInvalidRequest) + }, s3err.GetInvalidMpObjectSizeErr(mpuObjSizeHdr) } if val < 0 { - debuglogger.Logf("value for 'x-amz-mp-objects-size' header is less than 0: %v", val) + debuglogger.Logf("value for 'x-amz-mp-object-size' header is less than 0: %v", val) return &Response{ MetaOpts: &MetaOptions{ BucketOwner: parsedAcl.Owner, }, - }, s3err.GetInvalidMpObjectSizeErr(val) + }, s3err.GetNegatvieMpObjectSizeErr(val) } mpuObjectSize = &val diff --git a/s3api/controllers/object-post_test.go b/s3api/controllers/object-post_test.go index baca9f3e..3d96d7de 100644 --- a/s3api/controllers/object-post_test.go +++ b/s3api/controllers/object-post_test.go @@ -420,7 +420,7 @@ func TestS3ApiController_CompleteMultipartUpload(t *testing.T) { BucketOwner: "root", }, }, - err: s3err.GetAPIError(s3err.ErrInvalidRequest), + err: s3err.GetInvalidMpObjectSizeErr("invalid_mp_object_size"), }, }, { @@ -438,7 +438,7 @@ func TestS3ApiController_CompleteMultipartUpload(t *testing.T) { BucketOwner: "root", }, }, - err: s3err.GetInvalidMpObjectSizeErr(-4), + err: s3err.GetNegatvieMpObjectSizeErr(-4), }, }, { diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index 28444872..77397d95 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -51,7 +51,7 @@ func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { Acc: acct, Bucket: bucket, Object: key, - Action: auth.PutBucketTaggingAction, + Action: auth.PutObjectTaggingAction, IsBucketPublic: IsBucketPublic, }) if err != nil { @@ -233,7 +233,7 @@ func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { }, err } - if partNumber < 1 || partNumber > 10000 { + if partNumber < minPartNumber || partNumber > maxPartNumber { debuglogger.Logf("invalid part number: %d", partNumber) return &Response{ MetaOpts: &MetaOptions{ @@ -349,7 +349,7 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { }, err } - if partNumber < 1 || partNumber > 10000 { + if partNumber < minPartNumber || partNumber > maxPartNumber { debuglogger.Logf("invalid part number: %d", partNumber) return &Response{ MetaOpts: &MetaOptions{ @@ -389,7 +389,7 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") grantRead := ctx.Get("X-Amz-Grant-Read") grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") - granWrite := ctx.Get("X-Amz-Grant-Write") + grantWrite := ctx.Get("X-Amz-Grant-Write") grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) @@ -420,7 +420,7 @@ func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { Key: &key, GrantFullControl: &grantFullControl, GrantRead: &grantRead, - GrantWrite: &granWrite, + GrantWrite: &grantWrite, ACL: types.ObjectCannedACL(acl), GrantReadACP: &grantReadACP, GrantWriteACP: &grantWriteACP, @@ -524,7 +524,7 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { } if taggingDirective != "" && taggingDirective != types.TaggingDirectiveCopy && taggingDirective != types.TaggingDirectiveReplace { - debuglogger.Logf("invalid tagging direcrive: %v", taggingDirective) + debuglogger.Logf("invalid tagging directive: %v", taggingDirective) return &Response{ MetaOpts: &MetaOptions{ BucketOwner: parsedAcl.Owner, diff --git a/s3err/s3err.go b/s3err/s3err.go index 4dd1a374..c2c3cc22 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -894,7 +894,7 @@ func GetIncorrectMpObjectSizeErr(expected, actual int64) APIError { } } -func GetInvalidMpObjectSizeErr(val int64) APIError { +func GetNegatvieMpObjectSizeErr(val int64) APIError { return APIError{ Code: "InvalidRequest", Description: fmt.Sprintf("Value for x-amz-mp-object-size header is less than zero: '%v'", val), @@ -902,6 +902,14 @@ func GetInvalidMpObjectSizeErr(val int64) APIError { } } +func GetInvalidMpObjectSizeErr(val string) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("Value for x-amz-mp-object-size header is invalid: '%s'", val), + HTTPStatusCode: http.StatusBadRequest, + } +} + func CreateExceedingRangeErr(objSize int64) APIError { return APIError{ Code: "InvalidArgument", diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 65ac11f9..f2e1e023 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -11260,7 +11260,7 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.CompleteMultipartUpload(ctx, input) cancel() - if err := checkApiErr(err, s3err.GetInvalidMpObjectSizeErr(invMpuSize)); err != nil { + if err := checkApiErr(err, s3err.GetNegatvieMpObjectSizeErr(invMpuSize)); err != nil { return err }