From b7c758b06577df5270c90dbb107ab5c8dcf915a4 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Thu, 19 Jun 2025 22:08:54 +0400 Subject: [PATCH] feat: implements advanced routing for bucket POST and object PUT operations. Fixes #1036 Fixes the issue when calling a non-existing root endpoint(POST /) the gateway returns `NoSuchBucket`. Now it returns the correct `MethodNotAllowed` error. --- backend/posix/posix.go | 8 + s3api/controllers/base.go | 8 - s3api/controllers/base_test.go | 1001 ------------------------- s3api/controllers/bucket-post.go | 100 +++ s3api/controllers/object-put.go | 860 +++++++++++++++++++++ s3api/controllers/utilities.go | 1 + s3api/middlewares/acl-parser.go | 2 +- s3api/middlewares/router-utilities.go | 18 + s3api/router.go | 18 +- tests/integration/group-tests.go | 1 + 10 files changed, 999 insertions(+), 1018 deletions(-) delete mode 100644 s3api/controllers/base_test.go create mode 100644 s3api/controllers/bucket-post.go create mode 100644 s3api/controllers/object-put.go diff --git a/backend/posix/posix.go b/backend/posix/posix.go index d8660a2..c71874b 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -2419,6 +2419,10 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. if errors.Is(err, syscall.EDQUOT) { return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded) } + // Return the error itself, if it's an 's3err.APIError' + if _, ok := err.(s3err.APIError); ok { + return nil, err + } return nil, fmt.Errorf("write part data: %w", err) } @@ -2853,6 +2857,10 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3 if errors.Is(err, syscall.EDQUOT) { return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrQuotaExceeded) } + // Return the error itself, if it's an 's3err.APIError' + if _, ok := err.(s3err.APIError); ok { + return s3response.PutObjectOutput{}, err + } return s3response.PutObjectOutput{}, fmt.Errorf("write object data: %w", err) } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index f348798..34c36b4 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -72,13 +72,6 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs } } -func getstring(s *string) string { - if s == nil { - return "" - } - return *s -} - func getint64(i *int64) int64 { if i == nil { return 0 @@ -1974,7 +1967,6 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error { ctx.Status(apierr.HTTPStatusCode) return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", "")) } - fmt.Println(err, "------------") fmt.Fprintf(os.Stderr, "Internal Error, %v\n", err) ctx.Status(http.StatusInternalServerError) diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go deleted file mode 100644 index 65eb7e4..0000000 --- a/s3api/controllers/base_test.go +++ /dev/null @@ -1,1001 +0,0 @@ -// Copyright 2023 Versity Software -// This file is licensed under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package controllers - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "reflect" - "strings" - "testing" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/gofiber/fiber/v2" - "github.com/valyala/fasthttp" - "github.com/versity/versitygw/auth" - "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/s3api/utils" - "github.com/versity/versitygw/s3err" - "github.com/versity/versitygw/s3response" -) - -var ( - acl auth.ACL - acldata []byte -) - -func init() { - var err error - acldata, err = json.Marshal(acl) - if err != nil { - panic(err) - } -} - -func TestNew(t *testing.T) { - type args struct { - be backend.Backend - iam auth.IAMService - } - - be := backend.BackendUnsupported{} - - tests := []struct { - name string - args args - want S3ApiController - }{ - { - name: "Initialize S3 api controller", - args: args{ - be: be, - iam: &auth.IAMServiceInternal{}, - }, - want: S3ApiController{ - be: be, - iam: &auth.IAMServiceInternal{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("New() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestS3ApiController_PutBucketActions(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - - // Mock valid acl - acl := auth.ACL{Owner: "valid access"} - acldata, err := json.Marshal(acl) - if err != nil { - t.Errorf("Failed to parse the params: %v", err.Error()) - return - } - - body := ` - - - - - hell - - string - - - - hello - - - ` - - invOwnerBody := ` - - - hello - - - ` - - tagBody := ` - - - - organization - marketing - - - - ` - - versioningBody := ` - - Enabled - Enabled - - ` - - policyBody := `{ - "Statement": [ - { - "Effect": "Allow", - "Principal": "*", - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::my-bucket/*" - } - ] - } - ` - - objectLockBody := ` - - Enabled - - - GOVERNANCE - 2 - - - - ` - - ownershipBody := ` - - - BucketOwnerEnforced - - - ` - - invalidOwnershipBody := ` - - - invalid_value - - - ` - - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - PutBucketAclFunc: func(context.Context, string, []byte) error { - return nil - }, - CreateBucketFunc: func(context.Context, *s3.CreateBucketInput, []byte) error { - return nil - }, - PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error { - return nil - }, - PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error { - return nil - }, - PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error { - return nil - }, - PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error { - return nil - }, - PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error { - return nil - }, - GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) { - return types.ObjectOwnershipBucketOwnerPreferred, nil - }, - }, - } - // Mock ctx.Locals - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{Owner: "valid access"}) - return ctx.Next() - }) - app.Put("/:bucket", s3ApiController.PutBucketActions) - - // invalid acl case - invAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - invAclReq.Header.Set("X-Amz-Acl", "invalid") - - // invalid acl case 2 - errAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - errAclReq.Header.Set("X-Amz-Acl", "private") - errAclReq.Header.Set("X-Amz-Grant-Read", "hello") - - // PutBucketAcl incorrect bucket owner case - incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody)) - - // PutBucketAcl acl success - aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil) - aclSuccReq.Header.Set("X-Amz-Acl", "private") - - // Invalid acl body case - errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body)) - errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello") - - invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil) - invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Put-bucket-tagging-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-tagging-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)), - }, - wantErr: false, - statusCode: 204, - }, - { - name: "Put-bucket-ownership-controls-invalid-ownership", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-ownership-controls-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-lock-configuration-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-lock-configuration-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-versioning-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-versioning-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", strings.NewReader(versioningBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-policy-invalid-body", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-policy-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket?policy", strings.NewReader(policyBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-bucket-acl-invalid-acl", - app: app, - args: args{ - req: invAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-acl", - app: app, - args: args{ - req: errAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-acl-body", - app: app, - args: args{ - req: errAclBodyReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-incorrect-bucket-owner", - app: app, - args: args{ - req: incorrectBucketOwner, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-bucket-acl-success", - app: app, - args: args{ - req: aclSuccReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Create-bucket-invalid-acl-ownership-combination", - app: app, - args: args{ - req: invAclOwnershipReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Create-bucket-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.PutBucketActions() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.PutBucketActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func TestS3ApiController_PutActions(t *testing.T) { - type args struct { - req *http.Request - } - - body := ` - - - - - hell - - string - - - - hello - - - ` - tagBody := ` - - - - string - string - - - - ` - - //retentionBody := ` - // - // GOVERNANCE - // 2025-01-01T00:00:00Z - // - //` - - legalHoldBody := ` - - ON - - ` - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error { - return nil - }, - CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) { - return s3response.CopyObjectOutput{ - CopyObjectResult: &s3response.CopyObjectResult{}, - }, nil - }, - PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) { - return s3response.PutObjectOutput{}, nil - }, - UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) { - return &s3.UploadPartOutput{}, nil - }, - PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error { - return nil - }, - UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { - return s3response.CopyPartResult{}, nil - }, - PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error { - return nil - }, - PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error { - return nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }, - } - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Put("/:bucket/:key/*", s3ApiController.PutActions) - - // UploadPartCopy success - uploadPartCpyReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=12asd32&partNumber=3", nil) - uploadPartCpyReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // UploadPartCopy error case - uploadPartCpyErrReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=12asd32&partNumber=invalid", nil) - uploadPartCpyErrReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // CopyObject success - cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - - // CopyObject invalid checksum algorithm - cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject") - cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") - - // PutObjectAcl success - aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - aclReq.Header.Set("X-Amz-Acl", "private") - - // PutObjectAcl success grt case - aclGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - aclGrtReq.Header.Set("X-Amz-Grant-Read", "private") - - // invalid acl case 1 - invAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", nil) - invAclReq.Header.Set("X-Amz-Acl", "invalid") - - // invalid acl case 2 - errAclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", nil) - errAclReq.Header.Set("X-Amz-Acl", "private") - errAclReq.Header.Set("X-Amz-Grant-Read", "hello") - - // invalid body & grt case - invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)) - invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello") - - // PutObject invalid checksum algorithm - invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm") - - // PutObject invalid base64 checksum - invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64") - - // PutObject invalid crc32 - invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm") - - // PutObject invalid crc32c - invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==") - - // PutObject invalid sha1 - invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==") - - // PutObject invalid sha256 - invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm") - - // PutObject multiple checksum headers - mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=") - mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") - - // PutObject checksum algorithm and header mismatch - checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil) - checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1") - checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Put-object-part-error-case", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=abc&partNumber=invalid", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-part-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?uploadId=4&partNumber=3", nil), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Set-tags-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?tagging", strings.NewReader(tagBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "put-object-retention-invalid-request", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil), - }, - wantErr: false, - statusCode: 400, - }, - //{ - // name: "put-object-retention-success", - // app: app, - // args: args{ - // req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)), - // }, - // wantErr: false, - // statusCode: 200, - //}, - { - name: "put-legal-hold-invalid-request", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil), - }, - wantErr: false, - statusCode: 400, - }, - { - name: "put-legal-hold-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-invalid-acl", - app: app, - args: args{ - req: invAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-incorrect-acl", - app: app, - args: args{ - req: errAclReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-incorrect-acl-body-case", - app: app, - args: args{ - req: invAclBodyGrtReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Put-object-acl-success", - app: app, - args: args{ - req: aclReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-success-body-case", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body)), - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-acl-success-grt-case", - app: app, - args: args{ - req: aclGrtReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Upload-part-copy-invalid-part-number", - app: app, - args: args{ - req: uploadPartCpyErrReq, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Upload-part-copy-success", - app: app, - args: args{ - req: uploadPartCpyReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Copy-object-invalid-checksum-algorithm", - app: app, - args: args{ - req: cpyInvChecksumAlgo, - }, - wantErr: false, - statusCode: 400, - }, - { - name: "Copy-object-success", - app: app, - args: args{ - req: cpySrcReq, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Put-object-success", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key/key2", nil), - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.PutActions() %v error = %v, wantErr %v", - tt.name, err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.PutActions() %v statusCode = %v, wantStatusCode = %v", - tt.name, resp.StatusCode, tt.statusCode) - } - } -} - -func TestS3ApiController_DeleteObjects(t *testing.T) { - type args struct { - req *http.Request - } - - app := fiber.New() - s3ApiController := S3ApiController{ - be: &BackendMock{ - GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { - return acldata, nil - }, - DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) { - return s3response.DeleteResult{}, nil - }, - GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) { - return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound) - }, - }, - } - - app.Use(func(ctx *fiber.Ctx) error { - utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"}) - utils.ContextKeyIsRoot.Set(ctx, true) - utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{}) - return ctx.Next() - }) - app.Post("/:bucket", s3ApiController.DeleteObjects) - - // Valid request body - xmlBody := `body` - - request := httptest.NewRequest(http.MethodPost, "/my-bucket", strings.NewReader(xmlBody)) - request.Header.Set("Content-Type", "application/xml") - - tests := []struct { - name string - app *fiber.App - args args - wantErr bool - statusCode int - }{ - { - name: "Delete-Objects-success", - app: app, - args: args{ - req: request, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Delete-Objects-error", - app: app, - args: args{ - req: httptest.NewRequest(http.MethodPost, "/my-bucket", nil), - }, - wantErr: false, - statusCode: 400, - }, - } - for _, tt := range tests { - resp, err := tt.app.Test(tt.args.req) - - if (err != nil) != tt.wantErr { - t.Errorf("S3ApiController.DeleteObjects() error = %v, wantErr %v", err, tt.wantErr) - } - - if resp.StatusCode != tt.statusCode { - t.Errorf("S3ApiController.DeleteObjects() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode) - } - } -} - -func Test_XMLresponse(t *testing.T) { - type args struct { - ctx *fiber.Ctx - resp any - err error - } - - app := fiber.New() - ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) - - tests := []struct { - name string - args args - wantErr bool - statusCode int - }{ - { - name: "Internal-server-error", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrInternalError), - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Error-not-implemented", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrNotImplemented), - }, - wantErr: false, - statusCode: 501, - }, - { - name: "Invalid-request-body", - args: args{ - ctx: ctx, - resp: make(chan int), - err: nil, - }, - wantErr: true, - statusCode: 200, - }, - { - name: "Successful-response", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - }, - wantErr: false, - statusCode: 200, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := SendXMLResponse(tt.args.ctx, tt.args.resp, tt.args.err, &MetaOpts{}); (err != nil) != tt.wantErr { - t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) - } - - statusCode := tt.args.ctx.Response().StatusCode() - - if statusCode != tt.statusCode { - t.Errorf("response() %v code = %v, wantErr %v", tt.name, statusCode, tt.wantErr) - } - - tt.args.ctx.Status(http.StatusOK) - }) - } -} - -func Test_response(t *testing.T) { - type args struct { - ctx *fiber.Ctx - resp any - err error - opts *MetaOpts - } - - app := fiber.New() - ctx := app.AcquireCtx(&fasthttp.RequestCtx{}) - - tests := []struct { - name string - args args - wantErr bool - statusCode int - }{ - { - name: "Internal-server-error", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrInternalError), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Internal-server-error-not-api", - args: args{ - ctx: ctx, - resp: nil, - err: fmt.Errorf("custom error"), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 500, - }, - { - name: "Error-not-implemented", - args: args{ - ctx: ctx, - resp: nil, - err: s3err.GetAPIError(s3err.ErrNotImplemented), - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 501, - }, - { - name: "Successful-response", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - opts: &MetaOpts{}, - }, - wantErr: false, - statusCode: 200, - }, - { - name: "Successful-response-status-204", - args: args{ - ctx: ctx, - resp: "Valid response", - err: nil, - opts: &MetaOpts{ - Status: 204, - }, - }, - wantErr: false, - statusCode: 204, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := SendResponse(tt.args.ctx, tt.args.err, tt.args.opts); (err != nil) != tt.wantErr { - t.Errorf("response() %v error = %v, wantErr %v", tt.name, err, tt.wantErr) - } - - statusCode := tt.args.ctx.Response().StatusCode() - - if statusCode != tt.statusCode { - t.Errorf("response() %v code = %v, wantErr %v", tt.name, statusCode, tt.wantErr) - } - }) - } -} diff --git a/s3api/controllers/bucket-post.go b/s3api/controllers/bucket-post.go new file mode 100644 index 0000000..f9ce29c --- /dev/null +++ b/s3api/controllers/bucket-post.go @@ -0,0 +1,100 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + var dObj s3response.DeleteObjects + + err := xml.Unmarshal(ctx.Body(), &dObj) + if err != nil { + debuglogger.Logf("error unmarshalling delete objects: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + err = auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Action: auth.DeleteObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, dObj.Objects, bypass, IsBucketPublic, c.be) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.DeleteObjects(ctx.Context(), + &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: dObj.Objects, + }, + }) + return &Response{ + Data: res, + MetaOpts: &MetaOptions{ + Action: metrics.ActionDeleteObjects, + ObjectCount: int64(len(dObj.Objects)), + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectRemovedDeleteObjects, + }, + }, err +} diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go new file mode 100644 index 0000000..3e1de9f --- /dev/null +++ b/s3api/controllers/object-put.go @@ -0,0 +1,860 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controllers + +import ( + "encoding/xml" + "fmt" + "io" + "net/url" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/auth" + "github.com/versity/versitygw/metrics" + "github.com/versity/versitygw/s3api/debuglogger" + "github.com/versity/versitygw/s3api/utils" + "github.com/versity/versitygw/s3err" + "github.com/versity/versitygw/s3event" + "github.com/versity/versitygw/s3response" +) + +func (c S3ApiController) PutObjectTagging(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + tagging, err := utils.ParseTagging(ctx.Body(), utils.TagLimitObject) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutBucketTaggingAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectTagging(ctx.Context(), bucket, key, tagging) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectTagging, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectTaggingPut, + }, + }, err +} + +func (c S3ApiController) PutObjectRetention(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectRetentionAction, + IsBucketPublic: IsBucketPublic, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + if bypass { + policy, err := c.be.GetBucketPolicy(ctx.Context(), bucket) + if err != nil { + bypass = false + } else { + if err := auth.VerifyBucketPolicy(policy, acct.Access, bucket, key, auth.BypassGovernanceRetentionAction); err != nil { + bypass = false + } + } + } + + retention, err := auth.ParseObjectLockRetentionInput(ctx.Body()) + if err != nil { + debuglogger.Logf("failed to parse object lock configuration input: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = c.be.PutObjectRetention(ctx.Context(), bucket, key, versionId, bypass, retention) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectRetention, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectLegalHold(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + versionId := ctx.Query("versionId") + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + var legalHold types.ObjectLockLegalHold + if err := xml.Unmarshal(ctx.Body(), &legalHold); err != nil { + debuglogger.Logf("failed to parse request body: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + if legalHold.Status != types.ObjectLockLegalHoldStatusOff && legalHold.Status != types.ObjectLockLegalHoldStatusOn { + debuglogger.Logf("invalid legal hold status: %v", legalHold.Status) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrMalformedXML) + } + + if err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectLegalHoldAction, + IsBucketPublic: IsBucketPublic, + }); err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err := c.be.PutObjectLegalHold(ctx.Context(), bucket, key, versionId, legalHold.Status == types.ObjectLockLegalHoldStatusOn) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectLegalHold, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) UploadPart(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + partNumber := int32(ctx.QueryInt("partNumber", -1)) + uploadId := ctx.Query("uploadId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + contentLengthStr := ctx.Get("Content-Length") + if contentLengthStr == "" { + contentLengthStr = "0" + } + // Use decoded content length if available because the + // middleware will decode the chunked transfer encoding + decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") + if decodedLength != "" { + contentLengthStr = decodedLength + } + + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + debuglogger.Logf("err parsing checksum headers: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var body io.Reader + bodyi := utils.ContextKeyBodyReader.Get(ctx) + if bodyi != nil { + body = bodyi.(io.Reader) + } else { + body = ctx.Request().BodyStream() + } + + res, err := c.be.UploadPart(ctx.Context(), + &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + PartNumber: &partNumber, + ContentLength: &contentLength, + Body: body, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + }) + var headers map[string]*string + if err == nil { + headers = map[string]*string{ + "ETag": res.ETag, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + } + } + return &Response{ + Headers: headers, + MetaOpts: &MetaOptions{ + ContentLength: contentLength, + Action: metrics.ActionUploadPart, + BucketOwner: parsedAcl.Owner, + }, + }, err + +} + +func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") + copySrcRange := ctx.Get("X-Amz-Copy-Source-Range") + partNumber := int32(ctx.QueryInt("partNumber", -1)) + uploadId := ctx.Query("uploadId") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + cs := copySource + copySource, err := url.QueryUnescape(copySource) + if err != nil { + debuglogger.Logf("error unescaping copy source %q: %v", cs, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + if partNumber < 1 || partNumber > 10000 { + debuglogger.Logf("invalid part number: %d", partNumber) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) + } + + err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + resp, err := c.be.UploadPartCopy(ctx.Context(), + &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &key, + CopySource: ©Source, + PartNumber: &partNumber, + UploadId: &uploadId, + CopySourceRange: ©SrcRange, + }) + var headers map[string]*string + if err == nil && resp.CopySourceVersionId != "" { + headers = map[string]*string{ + "x-amz-copy-source-version-id": &resp.CopySourceVersionId, + } + } + return &Response{ + Headers: headers, + Data: resp, + MetaOpts: &MetaOptions{ + Action: metrics.ActionUploadPartCopy, + BucketOwner: parsedAcl.Owner, + }, + }, err +} + +func (c S3ApiController) PutObjectAcl(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + acl := ctx.Get("X-Amz-Acl") + grantFullControl := ctx.Get("X-Amz-Grant-Full-Control") + grantRead := ctx.Get("X-Amz-Grant-Read") + grantReadACP := ctx.Get("X-Amz-Grant-Read-Acp") + granWrite := ctx.Get("X-Amz-Grant-Write") + grantWriteACP := ctx.Get("X-Amz-Grant-Write-Acp") + grants := grantFullControl + grantRead + grantReadACP + granWrite + grantWriteACP + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAclAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var input *s3.PutObjectAclInput + if len(ctx.Body()) > 0 { + if grants+acl != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl)", grants, acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + var accessControlPolicy auth.AccessControlPolicy + err := xml.Unmarshal(ctx.Body(), &accessControlPolicy) + if err != nil { + debuglogger.Logf("error unmarshalling access control policy: %v", err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + //TODO: This part will be changed when object acls are implemented + + grants := []types.Grant{} + for _, grt := range accessControlPolicy.AccessControlList.Grants { + grants = append(grants, types.Grant{ + Grantee: &types.Grantee{ + ID: &grt.Grantee.ID, + Type: grt.Grantee.Type, + }, + Permission: types.Permission(grt.Permission), + }) + } + + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + ACL: "", + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: accessControlPolicy.Owner, + Grants: grants, + }, + } + } + if acl != "" { + if acl != "private" && acl != "public-read" && acl != "public-read-write" { + debuglogger.Logf("invalid acl: %q", acl) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + if len(ctx.Body()) > 0 || grants != "" { + debuglogger.Logf("invalid request: %q (grants) %q (acl) %v (body len)", grants, acl, len(ctx.Body())) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + ACL: types.ObjectCannedACL(acl), + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: &types.Owner{ID: &parsedAcl.Owner}, + }, + } + } + if grants != "" { + input = &s3.PutObjectAclInput{ + Bucket: &bucket, + Key: &key, + GrantFullControl: &grantFullControl, + GrantRead: &grantRead, + GrantReadACP: &grantReadACP, + GrantWrite: &granWrite, + GrantWriteACP: &grantWriteACP, + AccessControlPolicy: &types.AccessControlPolicy{ + Owner: &types.Owner{ID: &parsedAcl.Owner}, + }, + ACL: "", + } + } + + err = c.be.PutObjectAcl(ctx.Context(), input) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObjectAcl, + BucketOwner: parsedAcl.Owner, + EventName: s3event.EventObjectAclPut, + }, + }, err +} + +func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") + copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") + copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") + copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") + copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") + metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy))) + taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy))) + contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") + expires := ctx.Get("Expires") + tagging := ctx.Get("x-amz-tagging") + storageClass := ctx.Get("X-Amz-Storage-Class") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + + cs := copySource + copySource, err := url.QueryUnescape(copySource) + if err != nil { + debuglogger.Logf("error unescaping copy source %q: %v", cs, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + + err = auth.VerifyObjectCopyAccess(ctx.Context(), c.be, copySource, + auth.AccessOptions{ + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var mtime *time.Time + if copySrcModifSince != "" { + tm, err := time.Parse(iso8601Format, copySrcModifSince) + if err != nil { + debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + mtime = &tm + } + var umtime *time.Time + if copySrcUnmodifSince != "" { + tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) + if err != nil { + debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + umtime = &tm + } + + metadata := utils.GetUserMetaData(&ctx.Request().Header) + + if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace { + debuglogger.Logf("invalid metadata directive: %v", metaDirective) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidMetadataDirective) + } + + if taggingDirective != "" && taggingDirective != types.TaggingDirectiveCopy && taggingDirective != types.TaggingDirectiveReplace { + debuglogger.Logf("invalid tagging direcrive: %v", taggingDirective) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidTaggingDirective) + } + + checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + objLock, err := utils.ParsObjectLockHdrs(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + res, err := c.be.CopyObject(ctx.Context(), + s3response.CopyObjectInput{ + Bucket: &bucket, + Key: &key, + ContentType: &contentType, + ContentDisposition: &contentDisposition, + ContentEncoding: &contentEncoding, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + Tagging: &tagging, + TaggingDirective: taggingDirective, + CopySource: ©Source, + CopySourceIfMatch: ©SrcIfMatch, + CopySourceIfNoneMatch: ©SrcIfNoneMatch, + CopySourceIfModifiedSince: mtime, + CopySourceIfUnmodifiedSince: umtime, + ExpectedBucketOwner: &acct.Access, + Metadata: metadata, + MetadataDirective: metaDirective, + StorageClass: types.StorageClass(storageClass), + ChecksumAlgorithm: checksumAlgorithm, + ObjectLockRetainUntilDate: &objLock.RetainUntilDate, + ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ObjectLockMode: objLock.ObjectLockMode, + }) + + var etag *string + if err == nil { + etag = res.CopyObjectResult.ETag + } + + return &Response{ + Headers: map[string]*string{ + "x-amz-version-id": res.VersionId, + "x-amz-copy-source-version-id": res.CopySourceVersionId, + }, + Data: res.CopyObjectResult, + MetaOpts: &MetaOptions{ + Action: metrics.ActionCopyObject, + BucketOwner: parsedAcl.Owner, + ObjectETag: etag, + VersionId: res.VersionId, + EventName: s3event.EventObjectCreatedCopy, + }, + }, err +} + +func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { + bucket := ctx.Params("bucket") + key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) + contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") + contentDisposition := ctx.Get("Content-Disposition") + contentLanguage := ctx.Get("Content-Language") + cacheControl := ctx.Get("Cache-Control") + expires := ctx.Get("Expires") + tagging := ctx.Get("x-amz-tagging") + // context locals + acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) + isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) + parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL) + IsBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) + + // Content Length + contentLengthStr := ctx.Get("Content-Length") + if contentLengthStr == "" { + contentLengthStr = "0" + } + // Use decoded content length if available because the + // middleware will decode the chunked transfer encoding + decodedLength := ctx.Get("X-Amz-Decoded-Content-Length") + if decodedLength != "" { + contentLengthStr = decodedLength + } + + // load the meta headers + metadata := utils.GetUserMetaData(&ctx.Request().Header) + + err := auth.VerifyAccess(ctx.Context(), c.be, + auth.AccessOptions{ + Readonly: c.readonly, + Acl: parsedAcl, + AclPermission: auth.PermissionWrite, + IsRoot: isRoot, + Acc: acct, + Bucket: bucket, + Object: key, + Action: auth.PutObjectAction, + IsBucketPublic: IsBucketPublic, + }) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + err = auth.CheckObjectAccess(ctx.Context(), bucket, acct.Access, []types.ObjectIdentifier{{Key: &key}}, true, IsBucketPublic, c.be) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + debuglogger.Logf("error parsing content length %q: %v", contentLengthStr, err) + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + + objLock, err := utils.ParsObjectLockHdrs(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + algorithm, checksums, err := utils.ParseChecksumHeaders(ctx) + if err != nil { + return &Response{ + MetaOpts: &MetaOptions{ + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + }, + }, err + } + + var body io.Reader + bodyi := utils.ContextKeyBodyReader.Get(ctx) + if bodyi != nil { + body = bodyi.(io.Reader) + } else { + body = ctx.Request().BodyStream() + } + + res, err := c.be.PutObject(ctx.Context(), + s3response.PutObjectInput{ + Bucket: &bucket, + Key: &key, + ContentLength: &contentLength, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + ContentDisposition: &contentDisposition, + ContentLanguage: &contentLanguage, + CacheControl: &cacheControl, + Expires: &expires, + Metadata: metadata, + Body: body, + Tagging: &tagging, + ObjectLockRetainUntilDate: &objLock.RetainUntilDate, + ObjectLockMode: objLock.ObjectLockMode, + ObjectLockLegalHoldStatus: objLock.LegalHoldStatus, + ChecksumAlgorithm: algorithm, + ChecksumCRC32: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + }) + return &Response{ + Headers: map[string]*string{ + "ETag": &res.ETag, + "x-amz-checksum-crc32": res.ChecksumCRC32, + "x-amz-checksum-crc32c": res.ChecksumCRC32C, + "x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME, + "x-amz-checksum-sha1": res.ChecksumSHA1, + "x-amz-checksum-sha256": res.ChecksumSHA256, + "x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType), + "x-amz-version-id": &res.VersionID, + }, + MetaOpts: &MetaOptions{ + ContentLength: contentLength, + Action: metrics.ActionPutObject, + BucketOwner: parsedAcl.Owner, + ObjectETag: &res.ETag, + ObjectSize: contentLength, + EventName: s3event.EventObjectCreatedPut, + }, + }, err +} diff --git a/s3api/controllers/utilities.go b/s3api/controllers/utilities.go index 097b747..3e6d434 100644 --- a/s3api/controllers/utilities.go +++ b/s3api/controllers/utilities.go @@ -172,6 +172,7 @@ func ProcessResponse(handler Handler, s3logger s3log.AuditLogger, s3evnt s3event } } +// Sets the response headers func SetResponseHeaders(ctx *fiber.Ctx, headers map[string]*string) { if headers == nil { return diff --git a/s3api/middlewares/acl-parser.go b/s3api/middlewares/acl-parser.go index ba1fc21..20a4a56 100644 --- a/s3api/middlewares/acl-parser.go +++ b/s3api/middlewares/acl-parser.go @@ -38,7 +38,7 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe path := ctx.Path() pathParts := strings.Split(path, "/") bucket := pathParts[1] - if path == "/" && ctx.Method() == http.MethodGet { + if path == "/" { return ctx.Next() } if ctx.Method() == http.MethodPatch { diff --git a/s3api/middlewares/router-utilities.go b/s3api/middlewares/router-utilities.go index 2a09f25..4792910 100644 --- a/s3api/middlewares/router-utilities.go +++ b/s3api/middlewares/router-utilities.go @@ -19,6 +19,7 @@ import ( "github.com/versity/versitygw/s3api/utils" ) +// Evaluates/Matches the provided requst query params func MatchQueryArgs(args ...string) fiber.Handler { return func(ctx *fiber.Ctx) error { if utils.ContextKeySkip.IsSet(ctx) { @@ -34,6 +35,23 @@ func MatchQueryArgs(args ...string) fiber.Handler { } } +// Evaluates/Matches the requst header +func MatchHeader(key string) fiber.Handler { + return func(ctx *fiber.Ctx) error { + if utils.ContextKeySkip.IsSet(ctx) { + return ctx.Next() + } + + val := ctx.Get(key) + if val == "" { + utils.ContextKeySkip.Set(ctx, true) + } + + return ctx.Next() + } +} + +// Evaluates/Matches the requst query param and value func MatchQueryArgWithValue(key, val string) fiber.Handler { return func(ctx *fiber.Ctx) error { if utils.ContextKeySkip.IsSet(ctx) { diff --git a/s3api/router.go b/s3api/router.go index 8014e95..31aa0ce 100644 --- a/s3api/router.go +++ b/s3api/router.go @@ -94,7 +94,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ bucketRouter.Get("", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm)) // DeleteObjects action - bucketRouter.Post("", ctrl.DeleteObjects) + bucketRouter.Post("", middlewares.MatchQueryArgs("delete"), controllers.ProcessResponse(ctrl.DeleteObjects, logger, evs, mm)) // HeadObject objectRouter.Head("", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm)) @@ -118,13 +118,15 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ objectRouter.Post("", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.CompleteMultipartUpload, logger, evs, mm)) objectRouter.Post("", middlewares.MatchQueryArgs("uploads"), controllers.ProcessResponse(ctrl.CreateMultipartUpload, logger, evs, mm)) - // CopyObject action - // PutObject action - // UploadPart action - // UploadPartCopy action - // PutObjectTagging action - // PutObjectAcl action - app.Put("/:bucket/:key/*", ctrl.PutActions) + // PUT object operations + objectRouter.Put("", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutObjectTagging, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.PutObjectRetention, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.PutObjectLegalHold, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.PutObjectAcl, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.UploadPartCopy, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchQueryArgs("uploadId", "partNumber"), controllers.ProcessResponse(ctrl.UploadPart, logger, evs, mm)) + objectRouter.Put("", middlewares.MatchHeader("X-Amz-Copy-Source"), controllers.ProcessResponse(ctrl.CopyObject, logger, evs, mm)) + objectRouter.Put("", controllers.ProcessResponse(ctrl.PutObject, logger, evs, mm)) // Return MethodNotAllowed for all the unmatched routes app.All("*", controllers.ProcessResponse(ctrl.HandleUnmatch, logger, evs, mm)) diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 6435e73..a3a15f5 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -917,6 +917,7 @@ func GetIntTests() IntTests { "PutObject_with_object_lock": PutObject_with_object_lock, "PutObject_invalid_legal_hold": PutObject_invalid_legal_hold, "PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode, + "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, "PutObject_invalid_checksum_header": PutObject_invalid_checksum_header,