feat: implements advanced routing for object GET actions.

This commit is contained in:
niksis02
2025-06-17 21:01:16 +04:00
parent d2038ca973
commit 56d4e4aa3e
5 changed files with 548 additions and 822 deletions

View File

@@ -19,7 +19,6 @@ import (
"errors"
"fmt"
"io"
"math"
"net/http"
"net/url"
"os"
@@ -73,605 +72,6 @@ func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs
}
}
func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
bucket := ctx.Params("bucket")
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
uploadId := ctx.Query("uploadId")
partNumberMarker := ctx.Query("part-number-marker")
acceptRange := ctx.Get("Range")
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
versionId := ctx.Query("versionId")
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
path := ctx.Path()
if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
key = key + "/"
}
if ctx.Request().URI().QueryArgs().Has("tagging") {
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectTaggingAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
})
}
tags, err := c.be.GetObjectTagging(ctx.Context(), bucket, key)
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
})
}
res := s3response.Tagging{
TagSet: s3response.TagSet{Tags: []s3response.Tag{}},
}
for key, val := range tags {
res.TagSet.Tags = append(res.TagSet.Tags,
s3response.Tag{Key: key, Value: val})
}
return SendXMLResponse(ctx, res, nil,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
})
}
if ctx.Request().URI().QueryArgs().Has("retention") {
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectRetentionAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
})
}
data, err := c.be.GetObjectRetention(ctx.Context(), bucket, key, versionId)
if err != nil {
return SendXMLResponse(ctx, data, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
})
}
retention, err := auth.ParseObjectLockRetentionOutput(data)
return SendXMLResponse(ctx, retention, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
})
}
if ctx.Request().URI().QueryArgs().Has("legal-hold") {
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectLegalHoldAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectLegalHold,
BucketOwner: parsedAcl.Owner,
})
}
data, err := c.be.GetObjectLegalHold(ctx.Context(), bucket, key, versionId)
return SendXMLResponse(ctx, auth.ParseObjectLegalHoldOutput(data), err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectLegalHold,
BucketOwner: parsedAcl.Owner,
})
}
if uploadId != "" {
if partNumberMarker != "" {
n, err := strconv.Atoi(partNumberMarker)
if err != nil || n < 0 {
if err != nil && c.debug {
debuglogger.Logf("error parsing part number marker %q: %v",
partNumberMarker, err)
}
return SendResponse(ctx,
s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
}
mxParts := ctx.Query("max-parts")
maxParts, err := utils.ParseUint(mxParts)
if err != nil {
if c.debug {
debuglogger.Logf("error parsing max parts %q: %v",
mxParts, err)
}
return SendResponse(ctx,
s3err.GetAPIError(s3err.ErrInvalidMaxParts),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.ListMultipartUploadPartsAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
PartNumberMarker: &partNumberMarker,
MaxParts: &maxParts,
})
return SendXMLResponse(ctx, res, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
})
}
if ctx.Request().URI().QueryArgs().Has("acl") {
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionReadAcp,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectAclAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAcl,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{
Bucket: &bucket,
Key: &key,
})
return SendXMLResponse(ctx, res, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAcl,
BucketOwner: parsedAcl.Owner,
})
}
if ctx.Request().URI().QueryArgs().Has("attributes") {
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectAttributesAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
maxParts := ctx.Get("X-Amz-Max-Parts")
partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
maxPartsParsed, err := utils.ParseUint(maxParts)
if err != nil {
if c.debug {
debuglogger.Logf("error parsing max parts %q: %v",
maxParts, err)
}
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidMaxParts),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
attrs, err := utils.ParseObjectAttributes(ctx)
if err != nil {
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.GetObjectAttributes(ctx.Context(),
&s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &key,
PartNumberMarker: &partNumberMarker,
MaxParts: &maxPartsParsed,
VersionId: &versionId,
})
if err != nil {
hdrs := []utils.CustomHeader{}
if res.DeleteMarker != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-delete-marker",
Value: "true",
})
}
if getstring(res.VersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
hdrs := []utils.CustomHeader{}
if getstring(res.VersionId) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
})
}
if res.DeleteMarker != nil && *res.DeleteMarker {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-delete-marker",
Value: "true",
})
}
if res.LastModified != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Last-Modified",
Value: res.LastModified.UTC().Format(iso8601TimeFormatExtended),
})
}
utils.SetResponseHeaders(ctx, hdrs)
return SendXMLResponse(ctx, utils.FilterObjectAttributes(attrs, res), nil,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
})
}
action := auth.GetObjectAction
if versionId != "" {
action = auth.GetObjectVersionAction
}
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: action,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
})
}
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
if c.debug {
debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode)
}
return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
})
}
utils.ContextKeySkipResBodyLog.Set(ctx, true)
res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
if res != nil {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-delete-marker",
Value: "true",
},
{
Key: "Last-Modified",
Value: res.LastModified.UTC().Format(timefmt),
},
})
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionHeadObject,
BucketOwner: parsedAcl.Owner,
})
}
contentType := getstring(res.ContentType)
if contentType == "" {
contentType = defaultContentType
}
acceptRanges := getstring(res.AcceptRanges)
if acceptRanges == "" {
acceptRanges = "bytes"
}
hdrs := []utils.CustomHeader{
{
Key: "Content-Type",
Value: contentType,
},
{
Key: "ETag",
Value: getstring(res.ETag),
},
{
Key: "accept-ranges",
Value: acceptRanges,
},
}
if getstring(res.ContentDisposition) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Disposition",
Value: getstring(res.ContentDisposition),
})
}
if getstring(res.ContentEncoding) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Encoding",
Value: getstring(res.ContentEncoding),
})
}
if getstring(res.ContentLanguage) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Language",
Value: getstring(res.ContentLanguage),
})
}
if getstring(res.CacheControl) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Cache-Control",
Value: getstring(res.CacheControl),
})
}
if getstring(res.ExpiresString) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Expires",
Value: getstring(res.ExpiresString),
})
}
if getstring(res.ContentRange) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Range",
Value: getstring(res.ContentRange),
})
}
if res.LastModified != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Last-Modified",
Value: res.LastModified.UTC().Format(timefmt),
})
}
if res.TagCount != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(*res.TagCount),
})
}
if res.StorageClass != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
})
}
if res.ChecksumCRC32 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
}
if res.ChecksumCRC32C != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
}
if res.ChecksumSHA1 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
}
if res.ChecksumSHA256 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
// Set other response headers
utils.SetResponseHeaders(ctx, hdrs)
// Set version id header
if getstring(res.VersionId) != "" {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-version-id",
Value: getstring(res.VersionId),
},
})
}
status := http.StatusOK
if acceptRange != "" {
status = http.StatusPartialContent
}
if res.Body != nil {
// -1 will stream response body until EOF if content length not set
contentLen := -1
if res.ContentLength != nil {
if *res.ContentLength > int64(math.MaxInt) {
debuglogger.Logf("content length %v int overflow",
*res.ContentLength)
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRange),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
})
}
contentLen = int(*res.ContentLength)
}
utils.StreamResponseBody(ctx, res.Body, contentLen)
}
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
ContentLength: getint64(res.ContentLength),
BucketOwner: parsedAcl.Owner,
Status: status,
})
}
func getstring(s *string) string {
if s == nil {
return ""

View File

@@ -24,7 +24,6 @@ import (
"reflect"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -85,208 +84,6 @@ func TestNew(t *testing.T) {
}
}
func getPtr(val string) *string {
return &val
}
func TestS3ApiController_GetActions(t *testing.T) {
type args struct {
req *http.Request
}
now := time.Now()
app := fiber.New()
contentLength := int64(1000)
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListPartsFunc: func(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, nil
},
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, nil
},
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Metadata: map[string]string{"hello": "world"},
ContentType: getPtr("application/xml"),
ContentEncoding: getPtr("gzip"),
ETag: getPtr("98sda7f97sa9df798sd79f8as9df"),
ContentLength: &contentLength,
LastModified: &now,
StorageClass: "storage class",
}, nil
},
GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
return map[string]string{"hello": "world"}, nil
},
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
result, err := json.Marshal(types.ObjectLockRetention{
Mode: types.ObjectLockRetentionModeCompliance,
})
if err != nil {
return nil, err
}
return result, nil
},
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
result := true
return &result, nil
},
},
}
app.Use(func(ctx *fiber.Ctx) error {
utils.ContextKeyAccount.Set(ctx, auth.Account{Access: "valid access"})
utils.ContextKeyIsRoot.Set(ctx, true)
utils.ContextKeyParsedAcl.Set(ctx, auth.ACL{})
return ctx.Next()
})
app.Get("/:bucket/:key/*", s3ApiController.GetActions)
// GetObjectAttributes success case
getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Get-actions-get-tags-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key/key.json?tagging", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-retention-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-legal-hold-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-invalid-max-parts-string",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=invalid", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-invalid-max-parts-negative",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=-8", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-invalid-part-number-marker-string",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=invalid", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-invalid-part-number-marker-negative",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=-8", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-list-object-parts-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?uploadId=hello&max-parts=200&part-number-marker=23", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-acl-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key?acl", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-attributes-success",
app: app,
args: args{
req: getObjAttrs,
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-invalid-checksum-mode",
app: app,
args: args{
req: invalidChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-get-object-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil),
},
wantErr: false,
statusCode: 200,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("S3ApiController.GetActions() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("S3ApiController.GetActions() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
})
}
}
func TestS3ApiController_PutBucketActions(t *testing.T) {
type args struct {
req *http.Request

View File

@@ -0,0 +1,534 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/debuglogger"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
func (c S3ApiController) GetObjectTagging(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectTaggingAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
},
}, err
}
data, err := c.be.GetObjectTagging(ctx.Context(), bucket, key)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
},
}, err
}
tags := s3response.Tagging{
TagSet: s3response.TagSet{Tags: []s3response.Tag{}},
}
for key, val := range data {
tags.TagSet.Tags = append(tags.TagSet.Tags,
s3response.Tag{Key: key, Value: val})
}
return &Response{
Data: tags,
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectTagging,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) GetObjectRetention(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectRetentionAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
},
}, err
}
data, err := c.be.GetObjectRetention(ctx.Context(), bucket, key, versionId)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
},
}, err
}
retention, err := auth.ParseObjectLockRetentionOutput(data)
return &Response{
Data: retention,
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectRetention,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) GetObjectLegalHold(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectLegalHoldAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectLegalHold,
BucketOwner: parsedAcl.Owner,
},
}, err
}
data, err := c.be.GetObjectLegalHold(ctx.Context(), bucket, key, versionId)
return &Response{
Data: auth.ParseObjectLegalHoldOutput(data),
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectLegalHold,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) GetObjectAcl(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionReadAcp,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectAclAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAcl,
BucketOwner: parsedAcl.Owner,
},
}, err
}
res, err := c.be.GetObjectAcl(ctx.Context(), &s3.GetObjectAclInput{
Bucket: &bucket,
Key: &key,
})
return &Response{
Data: res,
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAcl,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) ListParts(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
uploadId := ctx.Query("uploadId")
partNumberMarker := ctx.Query("part-number-marker")
maxPartsStr := ctx.Query("max-parts")
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
// parse the part number marker
if partNumberMarker != "" {
n, err := strconv.Atoi(partNumberMarker)
if err != nil || n < 0 {
debuglogger.Logf("invalid part number marker %q: %v",
partNumberMarker, err)
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker)
}
}
// parse the max parts
maxParts, err := utils.ParseUint(maxPartsStr)
if err != nil {
debuglogger.Logf("error parsing max parts %q: %v",
maxPartsStr, err)
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidMaxParts)
}
err = auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.ListMultipartUploadPartsAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
},
}, err
}
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
PartNumberMarker: &partNumberMarker,
MaxParts: &maxParts,
})
return &Response{
Data: res,
MetaOpts: &MetaOptions{
Action: metrics.ActionListParts,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) GetObjectAttributes(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
maxPartsStr := ctx.Get("X-Amz-Max-Parts")
partNumberMarker := ctx.Get("X-Amz-Part-Number-Marker")
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: auth.GetObjectAttributesAction,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, err
}
// parse max parts
maxParts, err := utils.ParseUint(maxPartsStr)
if err != nil {
debuglogger.Logf("error parsing max parts %q: %v",
maxPartsStr, err)
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidMaxParts)
}
// parse the object attributes
attrs, err := utils.ParseObjectAttributes(ctx)
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, err
}
res, err := c.be.GetObjectAttributes(ctx.Context(),
&s3.GetObjectAttributesInput{
Bucket: &bucket,
Key: &key,
PartNumberMarker: &partNumberMarker,
MaxParts: &maxParts,
VersionId: &versionId,
})
if err != nil {
headers := map[string]*string{
"x-amz-version-id": res.VersionId,
}
if res.DeleteMarker != nil && *res.DeleteMarker {
headers["x-amz-delete-marker"] = utils.GetStringPtr("true")
}
return &Response{
Headers: headers,
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, err
}
headers := map[string]*string{
"x-amz-version-id": res.VersionId,
"Last-Modified": utils.FormatDatePtrToString(res.LastModified, iso8601TimeFormatExtended),
}
if res.DeleteMarker != nil && *res.DeleteMarker {
headers["x-amz-delete-marker"] = utils.GetStringPtr("true")
}
return &Response{
Headers: headers,
Data: utils.FilterObjectAttributes(attrs, res),
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, err
}
func (c S3ApiController) GetObject(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
acceptRange := ctx.Get("Range")
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
parsedAcl := utils.ContextKeyParsedAcl.Get(ctx).(auth.ACL)
isPublicBucket := utils.ContextKeyPublicBucket.IsSet(ctx)
utils.ContextKeySkipResBodyLog.Set(ctx, true)
action := auth.GetObjectAction
if versionId != "" {
action = auth.GetObjectVersionAction
}
err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{
Readonly: c.readonly,
Acl: parsedAcl,
AclPermission: auth.PermissionRead,
IsRoot: isRoot,
Acc: acct,
Bucket: bucket,
Object: key,
Action: action,
IsBucketPublic: isPublicBucket,
})
if err != nil {
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
},
}, err
}
// validate the checksum mode
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
debuglogger.Logf("invalid x-amz-checksum-mode header value: %v", checksumMode)
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode")
}
res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
var headers map[string]*string
if res != nil {
headers = map[string]*string{
"x-amz-delete-marker": utils.GetStringPtr("true"),
"Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt),
}
}
return &Response{
Headers: headers,
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObjectAttributes,
BucketOwner: parsedAcl.Owner,
},
}, err
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
status := http.StatusOK
if acceptRange != "" {
status = http.StatusPartialContent
}
if res.Body != nil {
// -1 will stream response body until EOF if content length not set
contentLen := -1
if res.ContentLength != nil {
if *res.ContentLength > int64(math.MaxInt) {
debuglogger.Logf("content length %v int overflow",
*res.ContentLength)
return &Response{
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObject,
ContentLength: getint64(res.ContentLength),
BucketOwner: parsedAcl.Owner,
Status: status,
},
}, s3err.GetAPIError(s3err.ErrInvalidRange)
}
contentLen = int(*res.ContentLength)
}
utils.StreamResponseBody(ctx, res.Body, contentLen)
}
return &Response{
Headers: map[string]*string{
"ETag": res.ETag,
"x-amz-restore": res.Restore,
"accept-ranges": res.AcceptRanges,
"Content-Range": res.ContentRange,
"Content-Disposition": res.ContentDisposition,
"Content-Encoding": res.ContentEncoding,
"Content-Language": res.ContentLanguage,
"Cache-Control": res.CacheControl,
"Expires": res.ExpiresString,
"x-amz-checksum-crc32": res.ChecksumCRC32,
"x-amz-checksum-crc64nvme": res.ChecksumCRC64NVME,
"x-amz-checksum-crc32c": res.ChecksumCRC32C,
"x-amz-checksum-sha1": res.ChecksumSHA1,
"x-amz-checksum-sha256": res.ChecksumSHA256,
"Content-Type": res.ContentType,
"x-amz-version-id": res.VersionId,
"Content-Length": utils.ConvertPtrToStringPtr(res.ContentLength),
"x-amz-mp-parts-count": utils.ConvertPtrToStringPtr(res.PartsCount),
"x-amz-tagging-count": utils.ConvertPtrToStringPtr(res.TagCount),
"x-amz-object-lock-mode": utils.ConvertToStringPtr(res.ObjectLockMode),
"x-amz-object-lock-legal-hold": utils.ConvertToStringPtr(res.ObjectLockLegalHoldStatus),
"x-amz-storage-class": utils.ConvertToStringPtr(res.StorageClass),
"x-amz-checksum-type": utils.ConvertToStringPtr(res.ChecksumType),
"x-amz-object-lock-retain-until-date": utils.FormatDatePtrToString(res.ObjectLockRetainUntilDate, time.RFC3339),
"Last-Modified": utils.FormatDatePtrToString(res.LastModified, timefmt),
},
MetaOpts: &MetaOptions{
Action: metrics.ActionGetObject,
ContentLength: getint64(res.ContentLength),
BucketOwner: parsedAcl.Owner,
Status: status,
},
}, nil
}

View File

@@ -15,6 +15,7 @@
package controllers
import (
"fmt"
"strings"
"time"
@@ -39,15 +40,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) (*Response, error) {
partNumberQuery := int32(ctx.QueryInt("partNumber", -1))
versionId := ctx.Query("versionId")
objRange := ctx.Get("Range")
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
if keyEnd != "" {
key = strings.Join([]string{key, keyEnd}, "/")
}
path := ctx.Path()
if path[len(path)-1:] == "/" && key[len(key)-1:] != "/" {
key = key + "/"
}
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
var partNumber *int32
if ctx.Request().URI().QueryArgs().Has("partNumber") {

View File

@@ -54,9 +54,10 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
}
// ListBuckets
app.Get("/", controllers.ProcessResponse(ctrl.ListBuckets, logger, evs, mm))
// Put bucket operations
// PUT bucket operations
app.Put("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.PutBucketTagging, logger, evs, mm))
app.Put("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.PutBucketOwnershipControls, logger, evs, mm))
app.Put("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.PutBucketVersioning, logger, evs, mm))
@@ -76,7 +77,7 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
app.Delete("/:bucket", middlewares.MatchQueryArgs("cors"), controllers.ProcessResponse(ctrl.DeleteBucketCors, logger, evs, mm))
app.Delete("/:bucket", controllers.ProcessResponse(ctrl.DeleteBucket, logger, evs, mm))
// Get bucket operations
// GET bucket operations
app.Get("/:bucket", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetBucketTagging, logger, evs, mm))
app.Get("/:bucket", middlewares.MatchQueryArgs("ownershipControls"), controllers.ProcessResponse(ctrl.GetBucketOwnershipControls, logger, evs, mm))
app.Get("/:bucket", middlewares.MatchQueryArgs("versioning"), controllers.ProcessResponse(ctrl.GetBucketVersioning, logger, evs, mm))
@@ -90,15 +91,16 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
app.Get("/:bucket", controllers.ProcessResponse(ctrl.ListObjects, logger, evs, mm))
// HeadObject
app.Head("/:bucket/:key/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm))
app.Head("/:bucket/*", controllers.ProcessResponse(ctrl.HeadObject, logger, evs, mm))
// GetObjectAcl action
// GetObject action
// ListObjectParts action
// GetObjectTagging action
// ListParts action
// GetObjectAttributes action
app.Get("/:bucket/:key/*", ctrl.GetActions)
// GET object operations
app.Get("/:bucket/*", middlewares.MatchQueryArgs("tagging"), controllers.ProcessResponse(ctrl.GetObjectTagging, logger, evs, mm))
app.Get("/:bucket/*", middlewares.MatchQueryArgs("retention"), controllers.ProcessResponse(ctrl.GetObjectRetention, logger, evs, mm))
app.Get("/:bucket/*", middlewares.MatchQueryArgs("legal-hold"), controllers.ProcessResponse(ctrl.GetObjectLegalHold, logger, evs, mm))
app.Get("/:bucket/*", middlewares.MatchQueryArgs("acl"), controllers.ProcessResponse(ctrl.GetObjectAcl, logger, evs, mm))
app.Get("/:bucket/*", middlewares.MatchQueryArgs("attributes"), controllers.ProcessResponse(ctrl.GetObjectAttributes, logger, evs, mm))
app.Get("/:bucket/*", middlewares.MatchQueryArgs("uploadId"), controllers.ProcessResponse(ctrl.ListParts, logger, evs, mm))
app.Get("/:bucket/*", controllers.ProcessResponse(ctrl.GetObject, logger, evs, mm))
// DeleteObject action
// AbortMultipartUpload action