From dfa1ed2358137af8c69b1b68ebc6329e423e5cbf Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 5 May 2025 22:31:55 +0400 Subject: [PATCH] fix: fixes the range parsing for GetObject. Adds range query support for HeadObject. Fixes #1258 Fixes #1257 Closes #1244 Adds range queries support for `HeadObject`. Fixes the range parsing logic for `GetObject`, which is used for `HeadObject` as well. Both actions follow the same rules for range parsing. Fixes the error message returned by `GetObject`. --- backend/azure/azure.go | 22 +- backend/common.go | 30 ++- backend/posix/posix.go | 38 +++- s3api/controllers/base.go | 14 ++ s3err/s3err.go | 2 +- tests/integration/group-tests.go | 10 +- tests/integration/tests.go | 363 +++++++++++++++---------------- tests/integration/utils.go | 14 +- 8 files changed, 270 insertions(+), 223 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 2ce0052..2c6e7db 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -418,7 +418,7 @@ func (az *Azure) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.G var opts *azblob.DownloadStreamOptions if *input.Range != "" { - offset, count, isValid, err := backend.ParseGetObjectRange(*resp.ContentLength, *input.Range) + offset, count, isValid, err := backend.ParseObjectRange(*resp.ContentLength, *input.Range) if err != nil { return nil, err } @@ -507,10 +507,26 @@ func (az *Azure) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3 if err != nil { return nil, azureErrToS3Err(err) } + var size int64 + if resp.ContentLength != nil { + size = *resp.ContentLength + } + + startOffset, length, isValid, err := backend.ParseObjectRange(size, getString(input.Range)) + if err != nil { + return nil, err + } + + var contentRange string + if isValid { + contentRange = fmt.Sprintf("bytes %v-%v/%v", + startOffset, startOffset+length-1, size) + } result := &s3.HeadObjectOutput{ - AcceptRanges: resp.AcceptRanges, - ContentLength: resp.ContentLength, + ContentRange: &contentRange, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, ContentType: resp.ContentType, ContentEncoding: resp.ContentEncoding, ContentLanguage: resp.ContentLanguage, diff --git a/backend/common.go b/backend/common.go index 741bb48..79bb7d2 100644 --- a/backend/common.go +++ b/backend/common.go @@ -88,11 +88,11 @@ var ( errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange) ) -// ParseGetObjectRange parses input range header and returns startoffset, length, isValid +// ParseObjectRange parses input range header and returns startoffset, length, isValid // and error. If no endoffset specified, then length is set to the object size // for invalid inputs, it returns no error, but isValid=false // `InvalidRange` error is returnd, only if startoffset is greater than the object size -func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) { +func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error) { if acceptRange == "" { return 0, size, false, nil } @@ -113,15 +113,17 @@ func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, er } startOffset, err := strconv.ParseInt(bRange[0], 10, 64) - if err != nil { + if err != nil && bRange[0] != "" { return 0, size, false, nil } - if startOffset >= size { - return 0, 0, false, errInvalidRange - } - if bRange[1] == "" { + if bRange[0] == "" { + return 0, size, false, nil + } + if startOffset >= size { + return 0, 0, false, errInvalidRange + } return startOffset, size - startOffset, true, nil } @@ -130,12 +132,22 @@ func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, er return 0, size, false, nil } - if endOffset < startOffset { + if startOffset > endOffset { return 0, size, false, nil } + // for ranges like 'bytes=-100' return the last bytes specified with 'endOffset' + if bRange[0] == "" { + endOffset = min(endOffset, size) + return size - endOffset, endOffset, true, nil + } + + if startOffset >= size { + return 0, 0, false, errInvalidRange + } + if endOffset >= size { - return startOffset, size - startOffset, true, nil + endOffset = size - 1 } return startOffset, endOffset - startOffset + 1, true, nil diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 1c3bbaa..28d815c 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -3364,9 +3364,6 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO if input.Key == nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if input.Range == nil { - return nil, s3err.GetAPIError(s3err.ErrInvalidRange) - } var versionId string if input.VersionId != nil { versionId = *input.VersionId @@ -3449,7 +3446,7 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } objSize := fi.Size() - startOffset, length, isValid, err := backend.ParseGetObjectRange(objSize, *input.Range) + startOffset, length, isValid, err := backend.ParseObjectRange(objSize, *input.Range) if err != nil { return nil, err } @@ -3635,20 +3632,34 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, fmt.Errorf("stat part: %w", err) } + size := part.Size() + + startOffset, length, isValid, err := backend.ParseObjectRange(size, getString(input.Range)) + if err != nil { + return nil, err + } + + var contentRange string + if isValid { + contentRange = fmt.Sprintf("bytes %v-%v/%v", + startOffset, startOffset+length-1, size) + } + b, err := p.meta.RetrieveAttribute(nil, bucket, partPath, etagkey) etag := string(b) if err != nil { etag = "" } partsCount := int32(len(ents)) - size := part.Size() return &s3.HeadObjectOutput{ + AcceptRanges: backend.GetPtrFromString("bytes"), LastModified: backend.GetTimePtr(part.ModTime()), ETag: &etag, PartsCount: &partsCount, - ContentLength: &size, + ContentLength: &length, StorageClass: types.StorageClassStandard, + ContentRange: &contentRange, }, nil } @@ -3740,6 +3751,17 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. size := fi.Size() + startOffset, length, isValid, err := backend.ParseObjectRange(size, getString(input.Range)) + if err != nil { + return nil, err + } + + var contentRange string + if isValid { + contentRange = fmt.Sprintf("bytes %v-%v/%v", + startOffset, startOffset+length-1, size) + } + var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus status, err := p.GetObjectLegalHold(ctx, bucket, object, versionId) if err == nil { @@ -3774,7 +3796,9 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } return &s3.HeadObjectOutput{ - ContentLength: &size, + ContentLength: &length, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentRange: &contentRange, ContentType: objMeta.ContentType, ContentEncoding: objMeta.ContentEncoding, ContentDisposition: objMeta.ContentDisposition, diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 9414be1..254d2f4 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -3207,6 +3207,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) versionId := ctx.Query("versionId") + objRange := ctx.Get("Range") key := ctx.Params("key") keyEnd := ctx.Params("*1") if keyEnd != "" { @@ -3277,6 +3278,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { PartNumber: partNumber, VersionId: &versionId, ChecksumMode: checksumMode, + Range: &objRange, }) if err != nil { if res != nil { @@ -3315,6 +3317,18 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: getstring(res.Restore), }, } + if getstring(res.AcceptRanges) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "accept-ranges", + Value: getstring(res.AcceptRanges), + }) + } + if getstring(res.ContentRange) != "" { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Range", + Value: getstring(res.ContentRange), + }) + } if getstring(res.ContentDisposition) != "" { headers = append(headers, utils.CustomHeader{ Key: "Content-Disposition", diff --git a/s3err/s3err.go b/s3err/s3err.go index bf121e7..64253e3 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -523,7 +523,7 @@ var errorCodeResponse = map[ErrorCode]APIError{ }, ErrInvalidRange: { Code: "InvalidRange", - Description: "The requested range is not valid for the request. Try another range.", + Description: "The requested range is not satisfiable", HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, }, ErrInvalidURI: { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 73888b1..6f00f84 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -168,6 +168,7 @@ func TestHeadObject(s *S3Conf) { HeadObject_directory_object_noslash(s) HeadObject_non_existing_dir_object(s) HeadObject_invalid_parent_dir(s) + HeadObject_with_range(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { HeadObject_not_enabled_checksum_mode(s) @@ -193,8 +194,7 @@ func TestGetObjectAttributes(s *S3Conf) { func TestGetObject(s *S3Conf) { GetObject_non_existing_key(s) GetObject_directory_object_noslash(s) - GetObject_should_succeed_for_invalid_ranges(s) - GetObject_content_ranges(s) + GetObject_with_range(s) GetObject_invalid_parent(s) GetObject_large_object(s) //TODO: remove the condition after implementing checksums in azure @@ -203,7 +203,6 @@ func TestGetObject(s *S3Conf) { } GetObject_success(s) GetObject_directory_success(s) - GetObject_by_range_success(s) GetObject_by_range_resp_status(s) GetObject_non_existing_dir_object(s) } @@ -877,6 +876,7 @@ func GetIntTests() IntTests { "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, "HeadObject_name_too_long": HeadObject_name_too_long, "HeadObject_invalid_parent_dir": HeadObject_invalid_parent_dir, + "HeadObject_with_range": HeadObject_with_range, "HeadObject_not_enabled_checksum_mode": HeadObject_not_enabled_checksum_mode, "HeadObject_checksums": HeadObject_checksums, "HeadObject_success": HeadObject_success, @@ -890,14 +890,12 @@ func GetIntTests() IntTests { "GetObjectAttributes_checksums": GetObjectAttributes_checksums, "GetObject_non_existing_key": GetObject_non_existing_key, "GetObject_directory_object_noslash": GetObject_directory_object_noslash, - "GetObject_should_succeed_for_invalid_ranges": GetObject_should_succeed_for_invalid_ranges, - "GetObject_content_ranges": GetObject_content_ranges, + "GetObject_with_range": GetObject_with_range, "GetObject_invalid_parent": GetObject_invalid_parent, "GetObject_large_object": GetObject_large_object, "GetObject_checksums": GetObject_checksums, "GetObject_success": GetObject_success, "GetObject_directory_success": GetObject_directory_success, - "GetObject_by_range_success": GetObject_by_range_success, "GetObject_by_range_resp_status": GetObject_by_range_resp_status, "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, "ListObjects_non_existing_bucket": ListObjects_non_existing_bucket, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index ff10ecd..e0de736 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -34,6 +34,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/versity/versitygw/s3err" "golang.org/x/sync/errgroup" ) @@ -3790,6 +3791,106 @@ func HeadObject_invalid_parent_dir(s *S3Conf) error { }) } +func HeadObject_with_range(s *S3Conf) error { + testName := "HeadObject_with_range" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, objLength := "my-obj", int64(2300) + _, err := putObjectWithData(objLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + testRange := func(rg, contentRange string, cLength int64, expectErr bool) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rg, + }) + cancel() + if err == nil && expectErr { + return fmt.Errorf("expected err 'RequestedRangeNotSatisfiable' error, instead got nil") + } + if err != nil { + if !expectErr { + return err + } + + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != "RequestedRangeNotSatisfiable" { + return fmt.Errorf("expected RequestedRangeNotSatisfiable, instead got %v", ae.ErrorCode()) + } + if ae.ErrorMessage() != "Requested Range Not Satisfiable" { + return fmt.Errorf("expected the error message to be 'Requested Range Not Satisfiable', instead got %v", ae.ErrorMessage()) + } + return nil + } + + return fmt.Errorf("invalid error got %w", err) + } + + if getString(res.AcceptRanges) != "bytes" { + return fmt.Errorf("expected accept ranges to be 'bytes', instead got %v", getString(res.AcceptRanges)) + } + if res.ContentLength == nil { + return fmt.Errorf("expected non nil content-length") + } + if *res.ContentLength != cLength { + return fmt.Errorf("expected content-length to be %v, instead got %v", cLength, *res.ContentLength) + } + if getString(res.ContentRange) != contentRange { + return fmt.Errorf("expected content-range to be %v, instead got %v", contentRange, getString(res.ContentRange)) + } + + return nil + } + + for _, el := range []struct { + objRange string + contentRange string + contentLength int64 + expectedErr bool + }{ + // invalid ranges: no error + {"100", "", objLength, false}, + {"100-", "", objLength, false}, + {"invalid_range", "", objLength, false}, + {"bytes=120", "", objLength, false}, + {"bytes=20-10", "", objLength, false}, + {"bytes=abc", "", objLength, false}, + {"bytes=abc-xyz", "", objLength, false}, + {"bytes=100-x", "", objLength, false}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), "", objLength, false}, + // valid ranges + {"bytes=-1000000", fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, false}, + {"bytes=100-", fmt.Sprintf("bytes 100-%v/%v", objLength-1, objLength), objLength - 100, false}, + {"bytes=-100", fmt.Sprintf("bytes %v-%v/%v", objLength-100, objLength-1, objLength), 100, false}, + {"bytes=0-", fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, false}, + {"bytes=100-200", fmt.Sprintf("bytes 100-200/%v", objLength), 101, false}, + {fmt.Sprintf("bytes=100-%v", objLength), fmt.Sprintf("bytes 100-%v/%v", objLength-1, objLength), objLength - 100, false}, + {fmt.Sprintf("bytes=0-%v", objLength), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, false}, + {fmt.Sprintf("bytes=0-%v", objLength-1), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, false}, + {fmt.Sprintf("bytes=-%v", objLength), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, false}, + + // not satisfiable ranges: return error + {fmt.Sprintf("bytes=%v-", objLength), "", 0, true}, + {fmt.Sprintf("bytes=%v-", objLength+2), "", 0, true}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength+100), "", 0, true}, + } { + err := testRange(el.objRange, el.contentRange, el.contentLength, el.expectedErr) + if err != nil { + return err + } + } + + return nil + }) +} + func HeadObject_success(s *S3Conf) error { testName := "HeadObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4205,12 +4306,11 @@ func GetObject_directory_object_noslash(s *S3Conf) error { }) } -func GetObject_invalid_range(s *S3Conf) error { - testName := "GetObject_invalid_range" +func GetObject_with_range(s *S3Conf) error { + testName := "GetObject_with_range" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dataLength, obj := int64(2500), "my-obj" - - _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + obj, objLength := "my-obj", int64(800) + res, err := putObjectWithData(objLength, &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, }, s3client) @@ -4218,27 +4318,90 @@ func GetObject_invalid_range(s *S3Conf) error { return err } - getobj := func(acceptRange string) error { + testGetObjectRange := func(rng, contentRange string, cLength int64, expData []byte, expErr error) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, Key: &obj, - Range: &acceptRange, + Range: &rng, }) cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { - return err + if err == nil && expErr != nil { + return fmt.Errorf("expected err %w, instead got nil", expErr) + } + if err != nil { + if expErr == nil { + return err + } + + parsedErr, ok := expErr.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type provided, expected s3err.APIError") + } + + return checkApiErr(err, parsedErr) + } + + if out.ContentLength == nil { + return fmt.Errorf("expected non nil content-length") + } + if *out.ContentLength != cLength { + return fmt.Errorf("expected content-length to be %v, instead got %v", cLength, *out.ContentLength) + } + if getString(out.AcceptRanges) != "bytes" { + return fmt.Errorf("expected accept-ranges to be 'bytes', instead got %v", getString(out.AcceptRanges)) + } + if getString(out.ContentRange) != contentRange { + return fmt.Errorf("expected content-range to be %v, instead got %v", contentRange, getString(out.ContentRange)) + } + + outData, err := io.ReadAll(out.Body) + if err != nil { + return fmt.Errorf("read object data: %w", err) + } + out.Body.Close() + + if !isSameData(outData, expData) { + return fmt.Errorf("incorrect data retrieved") } return nil } - for _, rg := range []string{ - "bytes=2500-3000", - "bytes=2501-4000", - "bytes=5000-", + for _, el := range []struct { + rng string + contentRange string + cLength int64 + expData []byte + expErr error + }{ + // invalid ranges: no error + {"100", "", objLength, res.data, nil}, + {"100-", "", objLength, res.data, nil}, + {"invalid_range", "", objLength, res.data, nil}, + {"bytes=120", "", objLength, res.data, nil}, + {"bytes=20-10", "", objLength, res.data, nil}, + {"bytes=abc", "", objLength, res.data, nil}, + {"bytes=abc-xyz", "", objLength, res.data, nil}, + {"bytes=100-x", "", objLength, res.data, nil}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), "", objLength, res.data, nil}, + // valid ranges + {"bytes=-1000000", fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, res.data, nil}, + {"bytes=100-", fmt.Sprintf("bytes 100-%v/%v", objLength-1, objLength), objLength - 100, res.data[100:], nil}, + {"bytes=-100", fmt.Sprintf("bytes %v-%v/%v", objLength-100, objLength-1, objLength), 100, res.data[objLength-100:], nil}, + {"bytes=0-", fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, res.data, nil}, + {"bytes=100-200", fmt.Sprintf("bytes 100-200/%v", objLength), 101, res.data[100:201], nil}, + {fmt.Sprintf("bytes=100-%v", objLength), fmt.Sprintf("bytes 100-%v/%v", objLength-1, objLength), objLength - 100, res.data[100:], nil}, + {fmt.Sprintf("bytes=0-%v", objLength), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, res.data, nil}, + {fmt.Sprintf("bytes=0-%v", objLength-1), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, res.data, nil}, + {fmt.Sprintf("bytes=-%v", objLength), fmt.Sprintf("bytes 0-%v/%v", objLength-1, objLength), objLength, res.data, nil}, + + // not satisfiable ranges: return error + {fmt.Sprintf("bytes=%v-", objLength), "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {fmt.Sprintf("bytes=%v-", objLength+2), "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, + {fmt.Sprintf("bytes=%v-%v", objLength+2, objLength+100), "", 0, nil, s3err.GetAPIError(s3err.ErrInvalidRange)}, } { - err := getobj(rg) + err := testGetObjectRange(el.rng, el.contentRange, el.cLength, el.expData, el.expErr) if err != nil { return err } @@ -4248,117 +4411,6 @@ func GetObject_invalid_range(s *S3Conf) error { }) } -func GetObject_should_succeed_for_invalid_ranges(s *S3Conf) error { - testName := "GetObject_should_succeed_for_invalid_ranges" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dataLength, obj := int64(1234567), "my-obj" - - _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - getObj := func(acceptRange string) error { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - Range: &acceptRange, - }) - cancel() - if err != nil { - return err - } - - if res.ContentLength == nil { - return fmt.Errorf("expected non nil content length") - } - if *res.ContentLength != dataLength { - return fmt.Errorf("expected Content-Length to be %v, instead got %v", - dataLength, *res.ContentLength) - } - if getString(res.ContentRange) != "" { - return fmt.Errorf("expected empty Content-Range, instead got %v", - *res.ContentRange) - } - if getString(res.AcceptRanges) != "bytes" { - return fmt.Errorf("expected the accept ranges to be 'bytes', instead got %v", - getString(res.AcceptRanges)) - } - - return nil - } - - for _, rg := range []string{ - "bytes=invalid-range", - "bytes=33-10", - "bytes-12-34", - "bytes=-2-5", - "byte=100-200", - "bytes=inv-300", - } { - err := getObj(rg) - if err != nil { - return err - } - } - - return nil - }) -} - -func GetObject_content_ranges(s *S3Conf) error { - testName := "GetObject_should_adjust_range_upper_limit" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dataLength, obj := int64(1024), "my-obj" - - _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - Range: getPtr("bytes=100-"), - }) - cancel() - if err != nil { - return err - } - - expectedRange := "bytes 100-1023/1024" - if getString(res.ContentRange) != expectedRange { - return fmt.Errorf("expected the accept ranges to be %v, instead got %v", - expectedRange, getString(res.ContentRange)) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - res, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - Range: getPtr("bytes=100-99999999"), - }) - cancel() - if err != nil { - return err - } - if getString(res.ContentRange) != expectedRange { - return fmt.Errorf("expected the accept ranges to be %v, instead got %v", - expectedRange, getString(res.ContentRange)) - } - - return nil - }) -} - func GetObject_invalid_parent(s *S3Conf) error { testName := "GetObject_invalid_parent" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4646,64 +4698,6 @@ func GetObject_directory_success(s *S3Conf) error { }) } -func GetObject_by_range_success(s *S3Conf) error { - testName := "GetObject_by_range_success" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - dataLength, obj := int64(1234567), "my-obj" - - r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }, s3client) - if err != nil { - return err - } - - for _, el := range []struct { - acceptRange string - contentRange string - startOffset int64 - endOffset int64 - }{ - {"bytes=100-200", fmt.Sprintf("bytes 100-200/%v", dataLength), 100, 201}, - {"bytes=100-", fmt.Sprintf("bytes 100-1234566/%v", dataLength), 100, dataLength}, - {"bytes=100-1234567", fmt.Sprintf("bytes 100-1234566/%v", dataLength), 100, dataLength}, - } { - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - Range: &el.acceptRange, - }) - defer cancel() - if err != nil { - return err - } - defer out.Body.Close() - - if getString(out.ContentRange) != el.contentRange { - return fmt.Errorf("expected content range: %v, instead got: %v", - el.contentRange, getString(out.ContentRange)) - } - if getString(out.AcceptRanges) != "bytes" { - return fmt.Errorf("expected accept range: bytes, instead got: %v", - getString(out.AcceptRanges)) - } - b, err := io.ReadAll(out.Body) - if err != nil && !errors.Is(err, io.EOF) { - return err - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, r.data[el.startOffset:el.endOffset]) { - return fmt.Errorf("data mismatch of range") - } - } - - return nil - }) -} - func GetObject_by_range_resp_status(s *S3Conf) error { testName := "GetObject_by_range_resp_status" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -8182,7 +8176,6 @@ func CreateMultipartUpload_with_tagging(s *S3Conf) error { } { err := testTagging(el.tagging, el.result, el.expectedErr) if err != nil { - fmt.Println("failing for: ", el.tagging) return err } } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 8028fe5..14c1e32 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -502,18 +502,8 @@ func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.Creat return out, err } -func isEqual(a, b []byte) bool { - if len(a) != len(b) { - return false - } - - for i, d := range a { - if d != b[i] { - return false - } - } - - return true +func isSameData(a, b []byte) bool { + return bytes.Equal(a, b) } func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {