diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index a17ebe9a..0b23ac5d 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -589,7 +589,16 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { } if res.Body != nil { - ctx.Response().SetBodyStream(res.Body, int(getint64(res.ContentLength))) + err := utils.StreamResponseBody(ctx, res.Body) + if err != nil { + SendResponse(ctx, nil, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionGetObject, + BucketOwner: parsedAcl.Owner, + }) + } } return SendResponse(ctx, nil, diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 17fd99fb..fe95285c 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -196,6 +196,30 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) { } } +// Streams the response body by chunks +func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser) error { + buf := make([]byte, 4096) // 4KB chunks + defer rdr.Close() + for { + n, err := rdr.Read(buf) + if n > 0 { + _, writeErr := ctx.Write(buf[:n]) + if writeErr != nil { + return fmt.Errorf("write chunk: %w", writeErr) + } + } + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return fmt.Errorf("read chunk: %w", err) + } + } + + return nil +} + func IsValidBucketName(bucket string) bool { if len(bucket) < 3 || len(bucket) > 63 { return false diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 81376001..1ecc64c1 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -172,6 +172,7 @@ func TestGetObject(s *S3Conf) { GetObject_invalid_ranges(s) GetObject_invalid_parent(s) GetObject_with_meta(s) + GetObject_large_object(s) GetObject_success(s) GetObject_directory_success(s) GetObject_by_range_success(s) @@ -753,6 +754,7 @@ func GetIntTests() IntTests { "GetObject_invalid_ranges": GetObject_invalid_ranges, "GetObject_invalid_parent": GetObject_invalid_parent, "GetObject_with_meta": GetObject_with_meta, + "GetObject_large_object": GetObject_large_object, "GetObject_success": GetObject_success, "GetObject_directory_success": GetObject_directory_success, "GetObject_by_range_success": GetObject_by_range_success, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 17483e89..049f3bb9 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -40,6 +40,7 @@ import ( var ( shortTimeout = 10 * time.Second + longTimeout = 60 * time.Second iso8601Format = "20060102T150405Z" nullVersionId = "null" ) @@ -3788,6 +3789,50 @@ func GetObject_with_meta(s *S3Conf) error { }) } +func GetObject_large_object(s *S3Conf) error { + testName := "GetObject_large_object" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + //FIXME: make the object size larger after + // resolving the context deadline exceeding issue + // in the github actions + dataLength, obj := int64(100*1024*1024), "my-obj" + ctype := defaultContentType + + r, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), longTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if *out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != r.csum { + return fmt.Errorf("expected the output data checksum to be %v, instead got %v", r.csum, outCsum) + } + return nil + }) +} + func GetObject_success(s *S3Conf) error { testName := "GetObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {