From d2038ca9736595964072f4701d2dd89090e5a735 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 16 Jun 2025 18:46:40 +0400 Subject: [PATCH] feat: implements advanced routing for HeadObject and bucket PUT operations. --- s3api/controllers/base.go | 260 -------------- s3api/controllers/base_test.go | 112 ------ s3api/controllers/bucket-head.go | 6 +- s3api/controllers/bucket-list.go | 14 + s3api/controllers/bucket-put.go | 582 +++++++++++++++++++++++++++++++ s3api/controllers/object-head.go | 160 +++++++++ s3api/controllers/utilities.go | 16 +- s3api/router.go | 13 +- s3api/utils/utils.go | 34 ++ 9 files changed, 813 insertions(+), 384 deletions(-) create mode 100644 s3api/controllers/bucket-put.go create mode 100644 s3api/controllers/object-head.go diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index b771fc5..3f8ef24 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -2538,266 +2538,6 @@ const ( timefmt = "Mon, 02 Jan 2006 15:04:05 GMT" ) -func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { - bucket := ctx.Params("bucket") - acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) - isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) - parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) - isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) - partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) - versionId := ctx.Query("versionId") - objRange := ctx.Get("Range") - key := ctx.Params("key") - keyEnd := ctx.Params("*1") - if keyEnd != "" { - key = strings.Join([]string{key, keyEnd}, "/") - } - path := ctx.Path() - if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { - key = key + "/" - } - - var partNumber *int32 - if ctx.Request().URI().QueryArgs().Has("partNumber") { - if partNumberQuery < 1 || partNumberQuery > 10000 { - if c.debug { - debuglogger.Logf("invalid part number: %d", partNumberQuery) - } - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - partNumber = &partNumberQuery - } - - err := auth.VerifyAccess(ctx.Context(), c.be, - auth.AccessOptions{ - Readonly: c.readonly, - Acl: parsedAcl, - AclPermission: auth.PermissionRead, - IsRoot: isRoot, - Acc: acct, - Bucket: bucket, - Object: key, - Action: auth.GetObjectAction, - IsBucketPublic: isPublicBucket, - }) - if err != nil { - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) - if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { - if c.debug { - debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) - } - return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"), - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - res, err := c.be.HeadObject(ctx.Context(), - &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &key, - PartNumber: partNumber, - VersionId: &versionId, - ChecksumMode: checksumMode, - Range: &objRange, - }) - if err != nil { - if res != nil { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-delete-marker", - Value: "true", - }, - { - Key: "Last-Modified", - Value: res.LastModified.UTC().Format(timefmt), - }, - }) - } - return SendResponse(ctx, err, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) - } - - utils.SetMetaHeaders(ctx, res.Metadata) - headers := []utils.CustomHeader{ - { - Key: "Content-Length", - Value: fmt.Sprint(getint64(res.ContentLength)), - }, - { - Key: "ETag", - Value: getstring(res.ETag), - }, - { - Key: "x-amz-restore", - Value: getstring(res.Restore), - }, - } - if getstring(res.AcceptRanges) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "accept-ranges", - Value: getstring(res.AcceptRanges), - }) - } - if getstring(res.ContentRange) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Range", - Value: getstring(res.ContentRange), - }) - } - if getstring(res.ContentDisposition) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Disposition", - Value: getstring(res.ContentDisposition), - }) - } - if getstring(res.ContentEncoding) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }) - } - if getstring(res.ContentLanguage) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Content-Language", - Value: getstring(res.ContentLanguage), - }) - } - if getstring(res.CacheControl) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Cache-Control", - Value: getstring(res.CacheControl), - }) - } - if getstring(res.ExpiresString) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "Expires", - Value: getstring(res.ExpiresString), - }) - } - if res.ObjectLockMode != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-mode", - Value: string(res.ObjectLockMode), - }) - } - if res.ObjectLockLegalHoldStatus != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-legal-hold", - Value: string(res.ObjectLockLegalHoldStatus), - }) - } - if res.ObjectLockRetainUntilDate != nil { - retainUntilDate := res.ObjectLockRetainUntilDate.Format(time.RFC3339) - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-object-lock-retain-until-date", - Value: retainUntilDate, - }) - } - if res.PartsCount != nil { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-mp-parts-count", - Value: fmt.Sprintf("%v", *res.PartsCount), - }) - } - if res.LastModified != nil { - lastmod := res.LastModified.UTC().Format(timefmt) - headers = append(headers, utils.CustomHeader{ - Key: "Last-Modified", - Value: lastmod, - }) - } - if res.StorageClass != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-storage-class", - Value: string(res.StorageClass), - }) - } - switch { - case res.ChecksumCRC32 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32", - Value: *res.ChecksumCRC32, - }) - case res.ChecksumCRC32C != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc32c", - Value: *res.ChecksumCRC32C, - }) - case res.ChecksumCRC64NVME != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-crc64nvme", - Value: *res.ChecksumCRC64NVME, - }) - case res.ChecksumSHA1 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha1", - Value: *res.ChecksumSHA1, - }) - case res.ChecksumSHA256 != nil: - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-sha256", - Value: *res.ChecksumSHA256, - }) - } - if res.ChecksumType != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-checksum-type", - Value: string(res.ChecksumType), - }) - } - - contentType := getstring(res.ContentType) - if contentType == "" { - contentType = defaultContentType - } - headers = append(headers, utils.CustomHeader{ - Key: "Content-Type", - Value: contentType, - }) - - if getstring(res.VersionId) != "" { - headers = append(headers, utils.CustomHeader{ - Key: "x-amz-version-id", - Value: getstring(res.VersionId), - }) - } - - utils.SetResponseHeaders(ctx, headers) - - return SendResponse(ctx, nil, - &MetaOpts{ - Logger: c.logger, - MetricsMng: c.mm, - Action: metrics.ActionHeadObject, - BucketOwner: parsedAcl.Owner, - }) -} - func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") key := ctx.Params("key") diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index d85fbc9..d619278 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -1154,118 +1154,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) { } } -func TestS3ApiController_HeadObject(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - - // Mock values - contentEncoding := "gzip" - contentType := "application/xml" - eTag := "Valid etag" - lastModifie := time.Now() - contentLength := int64(64) - - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { - return &s3.HeadObjectOutput{ - ContentEncoding: &contentEncoding, - ContentLength: &contentLength, - ContentType: &contentType, - LastModified: &lastModifie, - ETag: &eTag, - }, nil - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Head("/:bucket/:key/*", s3ApiController.HeadObject) - - //Error case - appErr := fiber.New() - - s3ApiControllerErr := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { - return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) - }, - }, - } - - appErr.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject) - - invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil) - invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Head-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Head-object-invalid-checksum-mode", - app: app, - args: args{ - req: invChecksumMode, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Head-object-error", - app: appErr, - args: args{ - req: httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil), - }, - wantErr: false, - statusCode: 400, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.HeadObject() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.HeadObject() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - func TestS3ApiController_CreateActions(t *testing.T) { type args struct { req *http.Request diff --git a/s3api/controllers/bucket-head.go b/s3api/controllers/bucket-head.go index 322d01e..bac2796 100644 --- a/s3api/controllers/bucket-head.go +++ b/s3api/controllers/bucket-head.go @@ -65,9 +65,9 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) (*Response, error) { } return &Response{ - Headers: map[string]string{ - "X-Amz-Access-Point-Alias": "false", - "X-Amz-Bucket-Region": region, + Headers: map[string]*string{ + "X-Amz-Access-Point-Alias": utils.GetStringPtr("false"), + "X-Amz-Bucket-Region": utils.GetStringPtr(region), }, MetaOpts: &MetaOptions{ Action: metrics.ActionHeadBucket, diff --git a/s3api/controllers/bucket-list.go b/s3api/controllers/bucket-list.go index 9dd649b..c2ceaa9 100644 --- a/s3api/controllers/bucket-list.go +++ b/s3api/controllers/bucket-list.go @@ -1,3 +1,17 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package controllers import ( diff --git a/s3api/controllers/bucket-put.go b/s3api/controllers/bucket-put.go new file mode 100644 index 0000000..5fa73c3 --- /dev/null +++ b/s3api/controllers/bucket-put.go @@ -0,0 +1,582 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) PutBucketTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketTaggingAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitBucket) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketTagging(ctx.Context(), bucket, tagging) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketTagging, + BucketOwner: parsedAcl.Owner, + Status: http.StatusNoContent, + }, + }, err +} + +func (c S3ApiController) PutBucketOwnershipControls(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketOwnershipControlsAction, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var ownershipControls s3response.OwnershipControls + if err := xml.Unmarshal(ctx.Body(), &ownershipControls); err != nil { + debuglogger.Logf("failed to unmarshal request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + rulesCount := len(ownershipControls.Rules) + isValidOwnership := utils.IsValidOwnership(ownershipControls.Rules[0].ObjectOwnership) + if rulesCount != 1 || !isValidOwnership { + if rulesCount != 1 { + debuglogger.Logf("ownership control rules should be 1, got %v", rulesCount) + } + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err := c.be.PutBucketOwnershipControls(ctx.Context(), bucket, ownershipControls.Rules[0].ObjectOwnership) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketOwnershipControls, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketVersioning(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketVersioningAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var versioningConf types.VersioningConfiguration + err = xml.Unmarshal(ctx.Body(), &versioningConf) + if err != nil { + debuglogger.Logf("error unmarshalling versioning configuration: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + if versioningConf.Status != types.BucketVersioningStatusEnabled && + versioningConf.Status != types.BucketVersioningStatusSuspended { + debuglogger.Logf("invalid versioning configuration status: %v", versioningConf.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + err = c.be.PutBucketVersioning(ctx.Context(), bucket, versioningConf.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketVersioning, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectLockConfiguration(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketObjectLockConfigurationAction, + IsBucketPublic: isPublicBucket, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + config, err := auth.ParseBucketLockConfigurationInput(ctx.Body()) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectLockConfiguration(ctx.Context(), bucket, config) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLockConfiguration, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketCors(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketCorsAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketCors(ctx.Context(), []byte{}) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketCors, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketPolicy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketPolicyAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.ValidatePolicyDocument(ctx.Body(), bucket, c.iam) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketPolicy(ctx.Context(), bucket, ctx.Body()) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketPolicy, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutBucketAcl(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + // context locals + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + var input *auth.PutBucketAclInput + + ownership, err := c.be.GetBucketOwnershipControls(ctx.Context(), bucket) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrOwnershipControlsNotFound)) { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + if ownership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled") + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrAclNotSupported) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWriteAcp, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.PutBucketAclAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if len(ctx.Body()) > 0 { + var accessControlPolicy auth.AccessControlPolicy + err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) + if err != nil { + debuglogger.Logf("error unmarshalling access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedACL) + } + + err = accessControlPolicy.Validate() + if err != nil { + debuglogger.Logf("invalid access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if *accessControlPolicy.Owner.ID != parsedAcl.Owner { + debuglogger.Logf("invalid access control policy owner id: %v, expected %v", *accessControlPolicy.Owner.ID, parsedAcl.Owner) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.APIError{ + Code: "InvalidArgument", + Description: "Invalid id", + HTTPStatusCode: http.StatusBadRequest, + } + } + + if grants+acl != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", + grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrUnexpectedContent) + } + + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &accessControlPolicy, + } + } else if acl != "" { + if acl != "private" && acl != "public-read" && acl != "public-read-write" { + debuglogger.Logf("invalid acl: %q", acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + if grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", + grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) + } + + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACL(acl), + } + } else if grants != "" { + input = &auth.PutBucketAclInput{ + Bucket: &bucket, + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + } + } else { + debuglogger.Logf("none of the bucket acl options has been specified: canned, req headers, req body") + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMissingSecurityHeader) + } + + updAcl, err := auth.UpdateACL(input, parsedAcl, c.iam, acct.Role == auth.RoleAdmin) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutBucketAcl(ctx.Context(), bucket, updAcl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutBucketAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) CreateBucket(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + lockEnabled := strings.EqualFold(ctx.Get("X-Amz-Bucket-Object-Lock-Enabled"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + objectOwnership := types.ObjectOwnership( + ctx.Get("X-Amz-Object-Ownership", string(types.ObjectOwnershipBucketOwnerEnforced)), + ) + + // validate the bucket name + if ok := utils.IsValidBucketName(bucket); !ok { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + }, + }, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + + // validate the object ownership value + if ok := utils.IsValidOwnership(objectOwnership); !ok { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + }, + }, s3err.APIError{ + Code: "InvalidArgument", + Description: fmt.Sprintf("Invalid x-amz-object-ownership header: %v", objectOwnership), + HTTPStatusCode: http.StatusBadRequest, + } + } + + if acl+grants != "" && objectOwnership == types.ObjectOwnershipBucketOwnerEnforced { + debuglogger.Logf("bucket acls are disabled for %v object ownership", objectOwnership) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, s3err.GetAPIError(s3err.ErrInvalidBucketAclWithObjectOwnership) + } + + if acl != "" && grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, s3err.GetAPIError(s3err.ErrBothCannedAndHeaderGrants) + } + + defACL := auth.ACL{ + Owner: acct.Access, + } + + updAcl, err := auth.UpdateACL(&auth.PutBucketAclInput{ + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + AccessControlPolicy: &auth.AccessControlPolicy{ + Owner: &types.Owner{ + ID: &acct.Access, + }}, + ACL: types.BucketCannedACL(acl), + }, defACL, c.iam, acct.Role == auth.RoleAdmin) + if err != nil { + debuglogger.Logf("failed to update bucket acl: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, err + } + + err = c.be.CreateBucket(ctx.Context(), &s3.CreateBucketInput{ + Bucket: &bucket, + ObjectOwnership: objectOwnership, + ObjectLockEnabledForBucket: &lockEnabled, + }, updAcl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCreateBucket, + BucketOwner: acct.Access, + }, + }, err +} diff --git a/s3api/controllers/object-head.go b/s3api/controllers/object-head.go new file mode 100644 index 0000000..a26b032 --- /dev/null +++ b/s3api/controllers/object-head.go @@ -0,0 +1,160 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" +) + +func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) { + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx) + // url values + bucket := ctx.Params("bucket") + partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) + versionId := ctx.Query("versionId") + objRange := ctx.Get("Range") + key := ctx.Params("key") + keyEnd := ctx.Params("*1") + if keyEnd != "" { + key = strings.Join([]string{key, keyEnd}, "/") + } + path := ctx.Path() + if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" { + key = key + "/" + } + + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < 1 || partNumberQuery > 10000 { + debuglogger.Logf("invalid part number: %d", partNumberQuery) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + partNumber = &partNumberQuery + } + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionRead, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.GetObjectAction, + IsBucketPublic: isPublicBucket, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode")) + if checksumMode != "" && checksumMode != types.ChecksumModeEnabled { + debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode") + } + + res, err := c.be.HeadObject(ctx.Context(), + &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, + VersionId: &versionId, + ChecksumMode: checksumMode, + Range: &objRange, + }) + if err != nil { + var headers map[string]*string + if res != nil { + headers = map[string]*string{ + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": utils.GetStringPtr(res.LastModified.UTC().Format(timefmt)), + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + // Set the metadata headers + utils.SetMetaHeaders(ctx, res.Metadata) + + return &Response{ + Headers: map[string]*string{ + "ETag": res.ETag, + "x-amz-restore": res.Restore, + "accept-ranges": res.AcceptRanges, + "Content-Range": res.ContentRange, + "Content-Disposition": res.ContentDisposition, + "Content-Encoding": res.ContentEncoding, + "Content-Language": res.ContentLanguage, + "Cache-Control": res.CacheControl, + "Expires": res.ExpiresString, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "Content-Type": res.ContentType, + "x-amz-version-id": res.VersionId, + "Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength), + "x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount), + "x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode), + "x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus), + "x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass), + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339), + "Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt), + }, + MetaOpts: &MetaOptions{ + Action: metrics.ActionHeadObject, + BucketOwner: parsedAcl.Owner, + }, + }, nil +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go index 97d66a7..f7b579e 100644 --- a/s3api/controllers/utilities.go +++ b/s3api/controllers/utilities.go @@ -50,7 +50,7 @@ type MetaOptions struct { type Response struct { Data any - Headers map[string]string + Headers map[string]*string MetaOpts *MetaOptions } @@ -65,6 +65,10 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } response, err := handler(ctx) + + // Set the response headers + SetResponseHeaders(ctx, response.Headers) + opts := response.MetaOpts // Send the metrics if mm != nil { @@ -98,9 +102,6 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event s3err.GetAPIError(s3err.ErrInternalError), "", "", "")) } - // Set the response headers - SetResponseHeaders(ctx, response.Headers) - if opts.Status == 0 { opts.Status = http.StatusOK } @@ -171,11 +172,14 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } } -func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]string) { +func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { if headers == nil { return } for key, val := range headers { - ctx.Response().Header.Add(key, val) + if val == nil || *val == "" { + continue + } + ctx.Response().Header.Add(key, *val) } } diff --git a/s3api/router.go b/s3api/router.go index e6150dd..70aaabd 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -57,7 +57,14 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm)) // Put bucket operations - app.Put("/:bucket", ctrl.PutBucketActions) + app.Put("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("object-lock"), controllers.ProcessResponse(ctrl.PutObjectLockConfiguration, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.PutBucketCors, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("policy"), controllers.ProcessResponse(ctrl.PutBucketPolicy, logger, evs, mm)) + app.Put("/:bucket", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutBucketAcl, logger, evs, mm)) + app.Put("/:bucket", controllers.ProcessResponse(ctrl.CreateBucket, logger, evs, mm)) // HeadBucket app.Head("/:bucket", controllers.ProcessResponse(ctrl.HeadBucket, logger, evs, mm)) @@ -82,8 +89,8 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ app.Get("/:bucket", middlewares.MatchQueryArgWithValue("list-type", "2"), controllers.ProcessResponse(ctrl.ListObjectsV2, logger, evs, mm)) app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) - // HeadObject action - app.Head("/:bucket/:key/*", ctrl.HeadObject) + // HeadObject + app.Head("/:bucket/:key/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) // GetObjectAcl action // GetObject action diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index ada265c..bf70fbf 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -664,3 +664,37 @@ func ParseTagging(data []byte, limit TagLimit) (map[string]string, error) { return tagSet, nil } + +// Returns the provided string pointer +func GetStringPtr(str string) *string { + if str == "" { + return nil + } + + return &str +} + +// Converts any type to a string pointer +func ConvertToStringPtr[T any](val T) *string { + str := fmt.Sprint(val) + return &str +} + +// Converst any pointer to a string pointer +func ConvertPtrToStringPtr[T any](val *T) *string { + if val == nil { + return nil + } + str := fmt.Sprint(*val) + return &str +} + +// Formats the date with the given formatting and returns a string pointer +func FormatDatePtrToString(date *time.Time, format string) *string { + if date == nil { + return nil + } + + formatted := date.UTC().Format(format) + return &formatted +}