diff --git a/backend/azure/azure.go b/backend/azure/azure.go index f9752ef1..8d408e3f 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -480,7 +480,7 @@ func (az *Azure) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAtt ETag: data.ETag, LastModified: data.LastModified, ObjectSize: data.ContentLength, - StorageClass: &data.StorageClass, + StorageClass: data.StorageClass, VersionId: data.VersionId, }, nil } diff --git a/backend/posix/posix.go b/backend/posix/posix.go index f9f7d015..850de7c2 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -941,9 +941,10 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl keyMarkerInd = len(uploads) } uploads = append(uploads, s3response.Upload{ - Key: objectName, - UploadID: uploadID, - Initiated: fi.ModTime(), + Key: objectName, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + Initiated: fi.ModTime(), }) } } @@ -1120,6 +1121,7 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon PartNumberMarker: partNumberMarker, Parts: parts, UploadID: uploadID, + StorageClass: types.StorageClassStandard, }, nil } @@ -1723,6 +1725,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO Metadata: userMetaData, TagCount: tagCount, ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, }, nil } @@ -1766,6 +1769,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO Metadata: userMetaData, TagCount: tagCount, ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, Body: &backend.FileSectionReadCloser{R: rdr, F: f}, }, nil } @@ -1820,6 +1824,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ETag: &etag, PartsCount: &partsCount, ContentLength: &size, + StorageClass: types.StorageClassStandard, }, nil } @@ -1896,6 +1901,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ObjectLockLegalHoldStatus: objectLockLegalHoldStatus, ObjectLockMode: objectLockMode, ObjectLockRetainUntilDate: objectLockRetainUntilDate, + StorageClass: types.StorageClassStandard, }, nil } @@ -1909,7 +1915,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr ETag: data.ETag, LastModified: data.LastModified, ObjectSize: data.ContentLength, - StorageClass: &data.StorageClass, + StorageClass: data.StorageClass, VersionId: data.VersionId, }, nil } @@ -1960,6 +1966,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr NextPartNumberMarker: resp.NextPartNumberMarker, Parts: parts, }, + StorageClass: types.StorageClassStandard, }, nil } @@ -2162,6 +2169,7 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { Key: &path, LastModified: &mtime, Size: &size, + StorageClass: types.ObjectStorageClassStandard, }, nil } @@ -2194,6 +2202,7 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { Key: &path, LastModified: &mtime, Size: &size, + StorageClass: types.ObjectStorageClassStandard, }, nil } } diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index c87dd841..a3f2f1a4 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -203,7 +203,7 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip ID: *u.Owner.ID, DisplayName: *u.Owner.DisplayName, }, - StorageClass: string(u.StorageClass), + StorageClass: u.StorageClass, Initiated: *u.Initiated, }) } @@ -270,7 +270,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re ID: *output.Owner.ID, DisplayName: *output.Owner.DisplayName, }, - StorageClass: string(output.StorageClass), + StorageClass: output.StorageClass, PartNumberMarker: pnm, NextPartNumberMarker: npmn, MaxParts: int(*output.MaxParts), @@ -362,7 +362,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt ETag: out.ETag, LastModified: out.LastModified, ObjectSize: out.ObjectSize, - StorageClass: &out.StorageClass, + StorageClass: out.StorageClass, VersionId: out.VersionId, ObjectParts: &parts, }, handleError(err) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 59be03f8..4fd6fa6e 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -419,56 +419,65 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { }) } - utils.SetMetaHeaders(ctx, res.Metadata) - var lastmod string - if res.LastModified != nil { - lastmod = res.LastModified.Format(timefmt) - } - contentType := getstring(res.ContentType) if contentType == "" { contentType = defaultContentType } + acceptRanges := getstring(res.AcceptRanges) + if acceptRanges == "" { + acceptRanges = "bytes" + } - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + hdrs := []utils.CustomHeader{ { Key: "Content-Type", Value: contentType, }, - { - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }, { Key: "ETag", Value: getstring(res.ETag), }, { - Key: "Last-Modified", - Value: lastmod, + Key: "accept-ranges", + Value: acceptRanges, }, - { - Key: "x-amz-storage-class", - Value: string(res.StorageClass), - }, - { + } + + if getstring(res.ContentRange) != "" { + hdrs = append(hdrs, utils.CustomHeader{ Key: "Content-Range", Value: getstring(res.ContentRange), - }, - { - Key: "accept-ranges", - Value: getstring(res.AcceptRanges), - }, - }) - - if res.TagCount != nil { - utils.SetResponseHeaders(ctx, []utils.CustomHeader{ - { - Key: "x-amz-tagging-count", - Value: fmt.Sprint(*res.TagCount), - }, }) } + if res.LastModified != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Last-Modified", + Value: res.LastModified.Format(timefmt), + }) + } + if getstring(res.ContentEncoding) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "Content-Encoding", + Value: getstring(res.ContentEncoding), + }) + } + if res.TagCount != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-tagging-count", + Value: fmt.Sprint(*res.TagCount), + }) + } + if res.StorageClass != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-storage-class", + Value: string(res.StorageClass), + }) + } + + // Set x-amz-meta-... headers + utils.SetMetaHeaders(ctx, res.Metadata) + // Set other response headers + utils.SetResponseHeaders(ctx, hdrs) status := http.StatusOK if acceptRange != "" { @@ -2751,10 +2760,6 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Key: "ETag", Value: getstring(res.ETag), }, - { - Key: "x-amz-storage-class", - Value: string(res.StorageClass), - }, { Key: "x-amz-restore", Value: getstring(res.Restore), @@ -2798,6 +2803,12 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: getstring(res.ContentEncoding), }) } + if res.StorageClass != "" { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-storage-class", + Value: string(res.StorageClass), + }) + } contentType := getstring(res.ContentType) if contentType == "" { diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 38b68a40..80553e96 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -259,7 +259,7 @@ func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3 output.ObjectSize = nil } if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok { - output.StorageClass = nil + output.StorageClass = "" } return output diff --git a/s3response/s3response.go b/s3response/s3response.go index 8c2c58a0..e6e413d0 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -57,7 +57,7 @@ type ListPartsResult struct { Owner Owner // The class of storage used to store the object. - StorageClass string + StorageClass types.StorageClass PartNumberMarker int NextPartNumberMarker int @@ -72,7 +72,7 @@ type GetObjectAttributesResult struct { ETag *string LastModified *time.Time ObjectSize *int64 - StorageClass *types.StorageClass + StorageClass types.StorageClass VersionId *string ObjectParts *ObjectParts } @@ -170,7 +170,7 @@ type Upload struct { UploadID string `xml:"UploadId"` Initiator Initiator Owner Owner - StorageClass string + StorageClass types.StorageClass Initiated time.Time } diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 4dcc5053..7a971e04 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -39,6 +39,7 @@ import ( var ( shortTimeout = 10 * time.Second iso8601Format = "20060102T150405Z" + emptyObjETag = "d41d8cd98f00b204e9800998ecf8427e" ) func Authentication_empty_auth_header(s *S3Conf) error { @@ -2984,6 +2985,9 @@ func HeadObject_mp_success(s *S3Conf) error { if *out.PartsCount != int32(partCount) { return fmt.Errorf("expected part count to be %v, instead got %v", partCount, *out.PartsCount) } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } return nil }) @@ -3065,6 +3069,9 @@ func HeadObject_success(s *S3Conf) error { if *out.ContentType != defaultContentType { return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType) } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } return nil }) @@ -3135,6 +3142,7 @@ func GetObjectAttributes_existing_object(s *S3Conf) error { ObjectAttributes: []types.ObjectAttributes{ types.ObjectAttributesEtag, types.ObjectAttributesObjectSize, + types.ObjectAttributesStorageClass, }, }) cancel() @@ -3157,6 +3165,9 @@ func GetObjectAttributes_existing_object(s *S3Conf) error { if out.Checksum != nil { return fmt.Errorf("expected checksum do be nil, instead got %v", *out.Checksum) } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } return nil }) @@ -3182,6 +3193,7 @@ func GetObjectAttributes_multipart_upload(s *S3Conf) error { Key: &obj, ObjectAttributes: []types.ObjectAttributes{ types.ObjectAttributesObjectParts, + types.ObjectAttributesStorageClass, }, }) cancel() @@ -3192,6 +3204,9 @@ func GetObjectAttributes_multipart_upload(s *S3Conf) error { if resp.ObjectParts == nil { return fmt.Errorf("expected non nil object parts") } + if resp.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, resp.StorageClass) + } for i, p := range resp.ObjectParts.Parts { if *p.PartNumber != *parts[i].PartNumber { @@ -3452,6 +3467,9 @@ func GetObject_success(s *S3Conf) error { if *out.ContentType != defaultContentType { return fmt.Errorf("expected content type %v, instead got %v", defaultContentType, *out.ContentType) } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } bdy, err := io.ReadAll(out.Body) if err != nil { @@ -3629,7 +3647,7 @@ func ListObjects_with_prefix(s *S3Conf) error { testName := "ListObjects_with_prefix" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { prefix := "obj" - objWithPrefix := []string{prefix + "/foo", prefix + "/bar", prefix + "/baz/bla"} + objWithPrefix := []string{prefix + "/bar", prefix + "/baz/bla", prefix + "/foo"} err := putObjects(s3client, append(objWithPrefix, []string{"xzy/csf", "hell"}...), bucket) if err != nil { return err @@ -3645,11 +3663,13 @@ func ListObjects_with_prefix(s *S3Conf) error { return err } + contents := createEmptyObjectsList(objWithPrefix) + if *out.Prefix != prefix { return fmt.Errorf("expected prefix %v, instead got %v", prefix, *out.Prefix) } - if !compareObjects(objWithPrefix, out.Contents) { - return fmt.Errorf("unexpected output for list objects with prefix") + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents) } return nil @@ -3687,8 +3707,9 @@ func ListObject_truncated(s *S3Conf) error { return fmt.Errorf("expected next-marker to be baz, instead got %v", *out1.NextMarker) } - if !compareObjects([]string{"bar", "baz"}, out1.Contents) { - return fmt.Errorf("unexpected output for list objects with max-keys") + contents := createEmptyObjectsList([]string{"bar", "baz"}) + if !compareObjects(contents, out1.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out1.Contents) } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) @@ -3709,8 +3730,10 @@ func ListObject_truncated(s *S3Conf) error { return fmt.Errorf("expected marker to be %v, instead got %v", *out1.NextMarker, *out2.Marker) } - if !compareObjects([]string{"foo"}, out2.Contents) { - return fmt.Errorf("unexpected output for list objects with max-keys") + contents = createEmptyObjectsList([]string{"foo"}) + + if !compareObjects(contents, out2.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out2.Contents) } return nil }) @@ -3840,8 +3863,10 @@ func ListObjects_marker_not_from_obj_list(s *S3Conf) error { return err } - if !compareObjects([]string{"foo", "qux", "hello", "xyz"}, out.Contents) { - return fmt.Errorf("expected output to be %v, instead got %v", []string{"foo", "qux", "hello", "xyz"}, out.Contents) + contents := createEmptyObjectsList([]string{"foo", "hello", "qux", "xyz"}) + + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected output to be %v, instead got %v", contents, out.Contents) } return nil @@ -3866,8 +3891,10 @@ func ListObjectsV2_start_after(s *S3Conf) error { return err } - if !compareObjects([]string{"baz", "foo"}, out.Contents) { - return fmt.Errorf("expected output to be %v, instead got %v", []string{"baz", "foo"}, out.Contents) + contents := createEmptyObjectsList([]string{"baz", "foo"}) + + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents) } return nil @@ -3905,8 +3932,10 @@ func ListObjectsV2_both_start_after_and_continuation_token(s *S3Conf) error { return fmt.Errorf("expected next-marker to be baz, instead got %v", *out.NextContinuationToken) } - if !compareObjects([]string{"bar"}, out.Contents) { - return fmt.Errorf("unexpected output for list objects with max-keys") + contents := createEmptyObjectsList([]string{"bar"}) + + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents) } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) @@ -3920,8 +3949,10 @@ func ListObjectsV2_both_start_after_and_continuation_token(s *S3Conf) error { return err } - if !compareObjects([]string{"foo", "quxx"}, resp.Contents) { - return fmt.Errorf("unexpected output for list objects with max-keys") + contents = createEmptyObjectsList([]string{"foo", "quxx"}) + + if !compareObjects(contents, resp.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, resp.Contents) } return nil @@ -3946,8 +3977,10 @@ func ListObjectsV2_start_after_not_in_list(s *S3Conf) error { return err } - if !compareObjects([]string{"foo", "quxx"}, out.Contents) { - return fmt.Errorf("expected output to be %v, instead got %v", []string{"foo", "quxx"}, out.Contents) + contents := createEmptyObjectsList([]string{"foo", "quxx"}) + + if !compareObjects(contents, out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, out.Contents) } return nil @@ -4062,7 +4095,9 @@ func ListObjectsV2_single_dir_object_with_delim_and_prefix(s *S3Conf) error { return err } - if !compareObjects([]string{"a/"}, res.Contents) { + contents := createEmptyObjectsList([]string{"a/"}) + + if !compareObjects(contents, res.Contents) { return fmt.Errorf("expected the object list to be %v, instead got %v", []string{"a/"}, res.Contents) } if len(res.CommonPrefixes) != 0 { @@ -4185,8 +4220,7 @@ func DeleteObject_success_status_code(s *S3Conf) error { func DeleteObjects_empty_input(s *S3Conf) error { testName := "DeleteObjects_empty_input" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - objects := []string{"foo", "bar", "baz"} - err := putObjects(s3client, objects, bucket) + err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) if err != nil { return err } @@ -4219,8 +4253,10 @@ func DeleteObjects_empty_input(s *S3Conf) error { return err } - if !compareObjects(objects, res.Contents) { - return fmt.Errorf("unexpected output for list objects with prefix") + contents := createEmptyObjectsList([]string{"bar", "baz", "foo"}) + + if !compareObjects(contents, res.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, res.Contents) } return nil @@ -4301,8 +4337,10 @@ func DeleteObjects_success(s *S3Conf) error { return err } - if !compareObjects(objects, res.Contents) { - return fmt.Errorf("unexpected output for list objects with prefix") + contents := createEmptyObjectsList(objects) + + if !compareObjects(contents, res.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", contents, res.Contents) } return nil @@ -5967,6 +6005,9 @@ func ListParts_success(s *S3Conf) error { return err } + if res.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, res.StorageClass) + } if ok := compareParts(parts, res.Parts); !ok { return fmt.Errorf("expected parts %+v, instead got %+v", parts, res.Parts) } @@ -6038,7 +6079,11 @@ func ListMultipartUploads_max_uploads(s *S3Conf) error { if err != nil { return err } - uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) + uploads = append(uploads, types.MultipartUpload{ + UploadId: out.UploadId, + Key: out.Key, + StorageClass: types.StorageClassStandard, + }) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) maxUploads := int32(2) @@ -6119,7 +6164,11 @@ func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error { if err != nil { return err } - uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) + uploads = append(uploads, types.MultipartUpload{ + UploadId: out.UploadId, + Key: out.Key, + StorageClass: types.StorageClassStandard, + }) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ @@ -6163,12 +6212,14 @@ func ListMultipartUploads_success(s *S3Conf) error { expected := []types.MultipartUpload{ { - Key: &obj1, - UploadId: out1.UploadId, + Key: &obj1, + UploadId: out1.UploadId, + StorageClass: types.StorageClassStandard, }, { - Key: &obj2, - UploadId: out2.UploadId, + Key: &obj2, + UploadId: out2.UploadId, + StorageClass: types.StorageClassStandard, }, } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index a4d3d9d1..c611d9b2 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -340,7 +340,13 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { return false } for i, item := range list1 { - if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId { + if *item.Key != *list2[i].Key { + return false + } + if *item.UploadId != *list2[i].UploadId { + return false + } + if item.StorageClass != list2[i].StorageClass { return false } } @@ -454,19 +460,22 @@ func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEnt return true } -func compareObjects(list1 []string, list2 []types.Object) bool { +func compareObjects(list1, list2 []types.Object) bool { if len(list1) != len(list2) { return false } - elementMap := make(map[string]bool) - - for _, elem := range list1 { - elementMap[elem] = true - } - - for _, elem := range list2 { - if _, found := elementMap[*elem.Key]; !found { + for i, obj := range list1 { + if *obj.Key != *list2[i].Key { + return false + } + if *obj.ETag != *list2[i].ETag { + return false + } + if *obj.Size != *list2[i].Size { + return false + } + if obj.StorageClass != list2[i].StorageClass { return false } } @@ -474,6 +483,22 @@ func compareObjects(list1 []string, list2 []types.Object) bool { return true } +// Creates a list of types.Object with the provided objects keys: objs []string +func createEmptyObjectsList(objs []string) (result []types.Object) { + size := int64(0) + for _, obj := range objs { + o := obj + result = append(result, types.Object{ + Key: &o, + Size: &size, + StorageClass: types.ObjectStorageClassStandard, + ETag: &emptyObjETag, + }) + } + + return +} + func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool { if len(list1) != len(list2) { return false