feat: Added the standard storage class to all the available get/list actions responses in posix. (#765)

This commit is contained in:
Jon Austin
2024-08-28 02:28:40 +04:00
committed by GitHub
parent 3ed7c18839
commit d79f978df9
8 changed files with 183 additions and 87 deletions

View File

@@ -480,7 +480,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt
ETag: data.ETag,
LastModified: data.LastModified,
ObjectSize: data.ContentLength,
StorageClass: &data.StorageClass,
StorageClass: data.StorageClass,
VersionId: data.VersionId,
}, nil
}

View File

@@ -941,9 +941,10 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
keyMarkerInd = len(uploads)
}
uploads = append(uploads, s3response.Upload{
Key: objectName,
UploadID: uploadID,
Initiated: fi.ModTime(),
Key: objectName,
UploadID: uploadID,
StorageClass: types.StorageClassStandard,
Initiated: fi.ModTime(),
})
}
}
@@ -1120,6 +1121,7 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
PartNumberMarker: partNumberMarker,
Parts: parts,
UploadID: uploadID,
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -1723,6 +1725,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
Metadata: userMetaData,
TagCount: tagCount,
ContentRange: &contentRange,
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -1766,6 +1769,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO
Metadata: userMetaData,
TagCount: tagCount,
ContentRange: &contentRange,
StorageClass: types.StorageClassStandard,
Body: &backend.FileSectionReadCloser{R: rdr, F: f},
}, nil
}
@@ -1820,6 +1824,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ETag: &etag,
PartsCount: &partsCount,
ContentLength: &size,
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -1896,6 +1901,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
ObjectLockMode: objectLockMode,
ObjectLockRetainUntilDate: objectLockRetainUntilDate,
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -1909,7 +1915,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
ETag: data.ETag,
LastModified: data.LastModified,
ObjectSize: data.ContentLength,
StorageClass: &data.StorageClass,
StorageClass: data.StorageClass,
VersionId: data.VersionId,
}, nil
}
@@ -1960,6 +1966,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr
NextPartNumberMarker: resp.NextPartNumberMarker,
Parts: parts,
},
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -2162,6 +2169,7 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc {
Key: &path,
LastModified: &mtime,
Size: &size,
StorageClass: types.ObjectStorageClassStandard,
}, nil
}
@@ -2194,6 +2202,7 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc {
Key: &path,
LastModified: &mtime,
Size: &size,
StorageClass: types.ObjectStorageClassStandard,
}, nil
}
}

View File

@@ -203,7 +203,7 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
ID: *u.Owner.ID,
DisplayName: *u.Owner.DisplayName,
},
StorageClass: string(u.StorageClass),
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
})
}
@@ -270,7 +270,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
ID: *output.Owner.ID,
DisplayName: *output.Owner.DisplayName,
},
StorageClass: string(output.StorageClass),
StorageClass: output.StorageClass,
PartNumberMarker: pnm,
NextPartNumberMarker: npmn,
MaxParts: int(*output.MaxParts),
@@ -362,7 +362,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
ETag: out.ETag,
LastModified: out.LastModified,
ObjectSize: out.ObjectSize,
StorageClass: &out.StorageClass,
StorageClass: out.StorageClass,
VersionId: out.VersionId,
ObjectParts: &parts,
}, handleError(err)

View File

@@ -419,56 +419,65 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
})
}
utils.SetMetaHeaders(ctx, res.Metadata)
var lastmod string
if res.LastModified != nil {
lastmod = res.LastModified.Format(timefmt)
}
contentType := getstring(res.ContentType)
if contentType == "" {
contentType = defaultContentType
}
acceptRanges := getstring(res.AcceptRanges)
if acceptRanges == "" {
acceptRanges = "bytes"
}
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
hdrs := []utils.CustomHeader{
{
Key: "Content-Type",
Value: contentType,
},
{
Key: "Content-Encoding",
Value: getstring(res.ContentEncoding),
},
{
Key: "ETag",
Value: getstring(res.ETag),
},
{
Key: "Last-Modified",
Value: lastmod,
Key: "accept-ranges",
Value: acceptRanges,
},
{
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
},
{
}
if getstring(res.ContentRange) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Range",
Value: getstring(res.ContentRange),
},
{
Key: "accept-ranges",
Value: getstring(res.AcceptRanges),
},
})
if res.TagCount != nil {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(*res.TagCount),
},
})
}
if res.LastModified != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Last-Modified",
Value: res.LastModified.Format(timefmt),
})
}
if getstring(res.ContentEncoding) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "Content-Encoding",
Value: getstring(res.ContentEncoding),
})
}
if res.TagCount != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(*res.TagCount),
})
}
if res.StorageClass != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
})
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
// Set other response headers
utils.SetResponseHeaders(ctx, hdrs)
status := http.StatusOK
if acceptRange != "" {
@@ -2751,10 +2760,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Key: "ETag",
Value: getstring(res.ETag),
},
{
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
},
{
Key: "x-amz-restore",
Value: getstring(res.Restore),
@@ -2798,6 +2803,12 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: getstring(res.ContentEncoding),
})
}
if res.StorageClass != "" {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-storage-class",
Value: string(res.StorageClass),
})
}
contentType := getstring(res.ContentType)
if contentType == "" {

View File

@@ -259,7 +259,7 @@ func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3
output.ObjectSize = nil
}
if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
output.StorageClass = nil
output.StorageClass = ""
}
return output

View File

@@ -57,7 +57,7 @@ type ListPartsResult struct {
Owner Owner
// The class of storage used to store the object.
StorageClass string
StorageClass types.StorageClass
PartNumberMarker int
NextPartNumberMarker int
@@ -72,7 +72,7 @@ type GetObjectAttributesResult struct {
ETag *string
LastModified *time.Time
ObjectSize *int64
StorageClass *types.StorageClass
StorageClass types.StorageClass
VersionId *string
ObjectParts *ObjectParts
}
@@ -170,7 +170,7 @@ type Upload struct {
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass string
StorageClass types.StorageClass
Initiated time.Time
}

View File

@@ -39,6 +39,7 @@ import (
var (
shortTimeout = 10 * time.Second
iso8601Format = "20060102T150405Z"
emptyObjETag = "d41d8cd98f00b204e9800998ecf8427e"
)
func Authentication_empty_auth_header(s *S3Conf) error {
@@ -2984,6 +2985,9 @@ func HeadObject_mp_success(s *S3Conf) error {
if *out.PartsCount != int32(partCount) {
return fmt.Errorf("expected part count to be %v, instead got %v", partCount, *out.PartsCount)
}
if out.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
}
return nil
})
@@ -3065,6 +3069,9 @@ func HeadObject_success(s *S3Conf) error {
if *out.ContentType != defaultContentType {
return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType)
}
if out.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
}
return nil
})
@@ -3135,6 +3142,7 @@ func GetObjectAttributes_existing_object(s *S3Conf) error {
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
types.ObjectAttributesObjectSize,
types.ObjectAttributesStorageClass,
},
})
cancel()
@@ -3157,6 +3165,9 @@ func GetObjectAttributes_existing_object(s *S3Conf) error {
if out.Checksum != nil {
return fmt.Errorf("expected checksum do be nil, instead got %v", *out.Checksum)
}
if out.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
}
return nil
})
@@ -3182,6 +3193,7 @@ func GetObjectAttributes_multipart_upload(s *S3Conf) error {
Key: &obj,
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectParts,
types.ObjectAttributesStorageClass,
},
})
cancel()
@@ -3192,6 +3204,9 @@ func GetObjectAttributes_multipart_upload(s *S3Conf) error {
if resp.ObjectParts == nil {
return fmt.Errorf("expected non nil object parts")
}
if resp.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, resp.StorageClass)
}
for i, p := range resp.ObjectParts.Parts {
if *p.PartNumber != *parts[i].PartNumber {
@@ -3452,6 +3467,9 @@ func GetObject_success(s *S3Conf) error {
if *out.ContentType != defaultContentType {
return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType)
}
if out.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass)
}
bdy, err := io.ReadAll(out.Body)
if err != nil {
@@ -3629,7 +3647,7 @@ func ListObjects_with_prefix(s *S3Conf) error {
testName := "ListObjects_with_prefix"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
prefix := "obj"
objWithPrefix := []string{prefix + "/foo", prefix + "/bar", prefix + "/baz/bla"}
objWithPrefix := []string{prefix + "/bar", prefix + "/baz/bla", prefix + "/foo"}
err := putObjects(s3client, append(objWithPrefix, []string{"xzy/csf", "hell"}...), bucket)
if err != nil {
return err
@@ -3645,11 +3663,13 @@ func ListObjects_with_prefix(s *S3Conf) error {
return err
}
contents := createEmptyObjectsList(objWithPrefix)
if *out.Prefix != prefix {
return fmt.Errorf("expected prefix %v, instead got %v", prefix, *out.Prefix)
}
if !compareObjects(objWithPrefix, out.Contents) {
return fmt.Errorf("unexpected output for list objects with prefix")
if !compareObjects(contents, out.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents)
}
return nil
@@ -3687,8 +3707,9 @@ func ListObject_truncated(s *S3Conf) error {
return fmt.Errorf("expected next-marker to be baz, instead got %v", *out1.NextMarker)
}
if !compareObjects([]string{"bar", "baz"}, out1.Contents) {
return fmt.Errorf("unexpected output for list objects with max-keys")
contents := createEmptyObjectsList([]string{"bar", "baz"})
if !compareObjects(contents, out1.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out1.Contents)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -3709,8 +3730,10 @@ func ListObject_truncated(s *S3Conf) error {
return fmt.Errorf("expected marker to be %v, instead got %v", *out1.NextMarker, *out2.Marker)
}
if !compareObjects([]string{"foo"}, out2.Contents) {
return fmt.Errorf("unexpected output for list objects with max-keys")
contents = createEmptyObjectsList([]string{"foo"})
if !compareObjects(contents, out2.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out2.Contents)
}
return nil
})
@@ -3840,8 +3863,10 @@ func ListObjects_marker_not_from_obj_list(s *S3Conf) error {
return err
}
if !compareObjects([]string{"foo", "qux", "hello", "xyz"}, out.Contents) {
return fmt.Errorf("expected output to be %v, instead got %v", []string{"foo", "qux", "hello", "xyz"}, out.Contents)
contents := createEmptyObjectsList([]string{"foo", "hello", "qux", "xyz"})
if !compareObjects(contents, out.Contents) {
return fmt.Errorf("expected output to be %v, instead got %v", contents, out.Contents)
}
return nil
@@ -3866,8 +3891,10 @@ func ListObjectsV2_start_after(s *S3Conf) error {
return err
}
if !compareObjects([]string{"baz", "foo"}, out.Contents) {
return fmt.Errorf("expected output to be %v, instead got %v", []string{"baz", "foo"}, out.Contents)
contents := createEmptyObjectsList([]string{"baz", "foo"})
if !compareObjects(contents, out.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents)
}
return nil
@@ -3905,8 +3932,10 @@ func ListObjectsV2_both_start_after_and_continuation_token(s *S3Conf) error {
return fmt.Errorf("expected next-marker to be baz, instead got %v", *out.NextContinuationToken)
}
if !compareObjects([]string{"bar"}, out.Contents) {
return fmt.Errorf("unexpected output for list objects with max-keys")
contents := createEmptyObjectsList([]string{"bar"})
if !compareObjects(contents, out.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents)
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
@@ -3920,8 +3949,10 @@ func ListObjectsV2_both_start_after_and_continuation_token(s *S3Conf) error {
return err
}
if !compareObjects([]string{"foo", "quxx"}, resp.Contents) {
return fmt.Errorf("unexpected output for list objects with max-keys")
contents = createEmptyObjectsList([]string{"foo", "quxx"})
if !compareObjects(contents, resp.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, resp.Contents)
}
return nil
@@ -3946,8 +3977,10 @@ func ListObjectsV2_start_after_not_in_list(s *S3Conf) error {
return err
}
if !compareObjects([]string{"foo", "quxx"}, out.Contents) {
return fmt.Errorf("expected output to be %v, instead got %v", []string{"foo", "quxx"}, out.Contents)
contents := createEmptyObjectsList([]string{"foo", "quxx"})
if !compareObjects(contents, out.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents)
}
return nil
@@ -4062,7 +4095,9 @@ func ListObjectsV2_single_dir_object_with_delim_and_prefix(s *S3Conf) error {
return err
}
if !compareObjects([]string{"a/"}, res.Contents) {
contents := createEmptyObjectsList([]string{"a/"})
if !compareObjects(contents, res.Contents) {
return fmt.Errorf("expected the object list to be %v, instead got %v", []string{"a/"}, res.Contents)
}
if len(res.CommonPrefixes) != 0 {
@@ -4185,8 +4220,7 @@ func DeleteObject_success_status_code(s *S3Conf) error {
func DeleteObjects_empty_input(s *S3Conf) error {
testName := "DeleteObjects_empty_input"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objects := []string{"foo", "bar", "baz"}
err := putObjects(s3client, objects, bucket)
err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket)
if err != nil {
return err
}
@@ -4219,8 +4253,10 @@ func DeleteObjects_empty_input(s *S3Conf) error {
return err
}
if !compareObjects(objects, res.Contents) {
return fmt.Errorf("unexpected output for list objects with prefix")
contents := createEmptyObjectsList([]string{"bar", "baz", "foo"})
if !compareObjects(contents, res.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, res.Contents)
}
return nil
@@ -4301,8 +4337,10 @@ func DeleteObjects_success(s *S3Conf) error {
return err
}
if !compareObjects(objects, res.Contents) {
return fmt.Errorf("unexpected output for list objects with prefix")
contents := createEmptyObjectsList(objects)
if !compareObjects(contents, res.Contents) {
return fmt.Errorf("expected the output to be %v, instead got %v", contents, res.Contents)
}
return nil
@@ -5967,6 +6005,9 @@ func ListParts_success(s *S3Conf) error {
return err
}
if res.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, res.StorageClass)
}
if ok := compareParts(parts, res.Parts); !ok {
return fmt.Errorf("expected parts %+v, instead got %+v", parts, res.Parts)
}
@@ -6038,7 +6079,11 @@ func ListMultipartUploads_max_uploads(s *S3Conf) error {
if err != nil {
return err
}
uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key})
uploads = append(uploads, types.MultipartUpload{
UploadId: out.UploadId,
Key: out.Key,
StorageClass: types.StorageClassStandard,
})
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
maxUploads := int32(2)
@@ -6119,7 +6164,11 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
if err != nil {
return err
}
uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key})
uploads = append(uploads, types.MultipartUpload{
UploadId: out.UploadId,
Key: out.Key,
StorageClass: types.StorageClassStandard,
})
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
@@ -6163,12 +6212,14 @@ func ListMultipartUploads_success(s *S3Conf) error {
expected := []types.MultipartUpload{
{
Key: &obj1,
UploadId: out1.UploadId,
Key: &obj1,
UploadId: out1.UploadId,
StorageClass: types.StorageClassStandard,
},
{
Key: &obj2,
UploadId: out2.UploadId,
Key: &obj2,
UploadId: out2.UploadId,
StorageClass: types.StorageClassStandard,
},
}

View File

@@ -340,7 +340,13 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
return false
}
for i, item := range list1 {
if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
if *item.Key != *list2[i].Key {
return false
}
if *item.UploadId != *list2[i].UploadId {
return false
}
if item.StorageClass != list2[i].StorageClass {
return false
}
}
@@ -454,19 +460,22 @@ func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEnt
return true
}
func compareObjects(list1 []string, list2 []types.Object) bool {
func compareObjects(list1, list2 []types.Object) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Key]; !found {
for i, obj := range list1 {
if *obj.Key != *list2[i].Key {
return false
}
if *obj.ETag != *list2[i].ETag {
return false
}
if *obj.Size != *list2[i].Size {
return false
}
if obj.StorageClass != list2[i].StorageClass {
return false
}
}
@@ -474,6 +483,22 @@ func compareObjects(list1 []string, list2 []types.Object) bool {
return true
}
// Creates a list of types.Object with the provided objects keys: objs []string
func createEmptyObjectsList(objs []string) (result []types.Object) {
size := int64(0)
for _, obj := range objs {
o := obj
result = append(result, types.Object{
Key: &o,
Size: &size,
StorageClass: types.ObjectStorageClassStandard,
ETag: &emptyObjETag,
})
}
return
}
func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
if len(list1) != len(list2) {
return false