From 481c9246c6cb87894ffadbebdebb966ff6ffad16 Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Fri, 3 May 2024 18:10:32 -0400 Subject: [PATCH] feat: HeadObject ation multipart upload case --- backend/posix/posix.go | 48 +++++++++++++++++-- s3api/controllers/base.go | 66 ++++++++++++++++++-------- s3err/s3err.go | 6 +++ tests/integration/group-tests.go | 6 +++ tests/integration/tests.go | 80 +++++++++++++++++++++++++++++++- 5 files changed, 182 insertions(+), 24 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index f38a413..40540a2 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -539,16 +539,16 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, return sum, nil } -func (p *Posix) retrieveUploadId(bucket, object string) (string, error) { +func (p *Posix) retrieveUploadId(bucket, object string) (string, [32]byte, error) { sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) entries, err := os.ReadDir(objdir) if err != nil || len(entries) == 0 { - return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } - return entries[0].Name(), nil + return entries[0].Name(), sum, nil } // fll out the user metadata map with the metadata for the object @@ -1536,6 +1536,46 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. bucket := *input.Bucket object := *input.Key + if input.PartNumber != nil { + uploadId, sum, err := p.retrieveUploadId(bucket, object) + if err != nil { + return nil, err + } + + ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId)) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return nil, fmt.Errorf("read parts: %w", err) + } + + partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber)) + + part, err := os.Stat(filepath.Join(bucket, partPath)) + if errors.Is(err, fs.ErrNotExist) { + return nil, s3err.GetAPIError(s3err.ErrInvalidPart) + } + if err != nil { + return nil, fmt.Errorf("stat part: %w", err) + } + + b, err := p.meta.RetrieveAttribute(bucket, partPath, etagkey) + etag := string(b) + if err != nil { + etag = "" + } + partsCount := int32(len(ents)) + size := part.Size() + + return &s3.HeadObjectOutput{ + LastModified: backend.GetTimePtr(part.ModTime()), + ETag: &etag, + PartsCount: &partsCount, + ContentLength: &size, + }, nil + } + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1618,7 +1658,7 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr return s3response.GetObjectAttributesResult{}, err } - uploadId, err := p.retrieveUploadId(*input.Bucket, *input.Key) + uploadId, _, err := p.retrieveUploadId(*input.Bucket, *input.Key) if err != nil { return s3response.GetObjectAttributesResult{}, err } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 6d7890a..e961d1a 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1421,7 +1421,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { log.Printf("invalid part number: %d", partNumber) } return SendXMLResponse(ctx, nil, - s3err.GetAPIError(s3err.ErrInvalidPart), + s3err.GetAPIError(s3err.ErrInvalidPartNumber), &MetaOpts{ Logger: c.logger, Action: "UploadPartCopy", @@ -2214,12 +2214,30 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { acct := ctx.Locals("account").(auth.Account) isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) + partNumberQuery := int32(ctx.QueryInt("partNumber", -1)) key := ctx.Params("key") keyEnd := ctx.Params("*1") if keyEnd != "" { key = strings.Join([]string{key, keyEnd}, "/") } + var partNumber *int32 + if ctx.Request().URI().QueryArgs().Has("partNumber") { + if partNumberQuery < 1 || partNumberQuery > 10000 { + if c.debug { + log.Printf("invalid part number: %d", partNumberQuery) + } + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumber), + &MetaOpts{ + Logger: c.logger, + Action: "HeadObject", + BucketOwner: parsedAcl.Owner, + }) + } + + partNumber = &partNumberQuery + } + err := auth.VerifyAccess(ctx.Context(), c.be, auth.AccessOptions{ Acl: parsedAcl, @@ -2241,8 +2259,9 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { res, err := c.be.HeadObject(ctx.Context(), &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &key, + Bucket: &bucket, + Key: &key, + PartNumber: partNumber, }) if err != nil { return SendResponse(ctx, err, @@ -2262,31 +2281,15 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { } utils.SetMetaHeaders(ctx, res.Metadata) - var lastmod string - if res.LastModified != nil { - lastmod = res.LastModified.Format(timefmt) - } headers := []utils.CustomHeader{ { Key: "Content-Length", Value: fmt.Sprint(getint64(res.ContentLength)), }, - { - Key: "Content-Type", - Value: getstring(res.ContentType), - }, - { - Key: "Content-Encoding", - Value: getstring(res.ContentEncoding), - }, { Key: "ETag", Value: getstring(res.ETag), }, - { - Key: "Last-Modified", - Value: lastmod, - }, { Key: "x-amz-storage-class", Value: string(res.StorageClass), @@ -2315,6 +2318,31 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: retainUntilDate, }) } + if res.PartsCount != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-mp-parts-count", + Value: fmt.Sprintf("%v", *res.PartsCount), + }) + } + if res.LastModified != nil { + lastmod := res.LastModified.Format(timefmt) + headers = append(headers, utils.CustomHeader{ + Key: "Last-Modified", + Value: lastmod, + }) + } + if res.ContentEncoding != nil { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Encoding", + Value: getstring(res.ContentEncoding), + }) + } + if res.ContentType != nil { + headers = append(headers, utils.CustomHeader{ + Key: "Content-Type", + Value: getstring(res.ContentType), + }) + } utils.SetResponseHeaders(ctx, headers) return SendResponse(ctx, nil, diff --git a/s3err/s3err.go b/s3err/s3err.go index f579b7b..2a14e89 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -69,6 +69,7 @@ const ( ErrInvalidMaxParts ErrInvalidPartNumberMarker ErrInvalidPart + ErrInvalidPartNumber ErrInternalError ErrInvalidCopyDest ErrInvalidCopySource @@ -209,6 +210,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidPartNumber: { + Code: "InvalidArgument", + Description: "Part number must be an integer between 1 and 10000, inclusive", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidCopyDest: { Code: "InvalidRequest", Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.", diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index a64973c..66bbbf2 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -104,6 +104,9 @@ func TestPutObject(s *S3Conf) { func TestHeadObject(s *S3Conf) { HeadObject_non_existing_object(s) + HeadObject_invalid_part_number(s) + HeadObject_non_existing_mp(s) + HeadObject_mp_success(s) HeadObject_success(s) } @@ -488,6 +491,9 @@ func GetIntTests() IntTests { "PutObject_invalid_long_tags": PutObject_invalid_long_tags, "PutObject_success": PutObject_success, "HeadObject_non_existing_object": HeadObject_non_existing_object, + "HeadObject_invalid_part_number": HeadObject_invalid_part_number, + "HeadObject_non_existing_mp": HeadObject_non_existing_mp, + "HeadObject_mp_success": HeadObject_mp_success, "HeadObject_success": HeadObject_success, "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 9be7f88..2a24b7d 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -2570,6 +2570,84 @@ func HeadObject_non_existing_object(s *S3Conf) error { }) } +func HeadObject_invalid_part_number(s *S3Conf) error { + testName := "HeadObject_invalid_part_number" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + partNumber := int32(-3) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + PartNumber: &partNumber, + }) + cancel() + if err := checkSdkApiErr(err, "BadRequest"); err != nil { + return err + } + return nil + }) +} + +func HeadObject_non_existing_mp(s *S3Conf) error { + testName := "HeadObject_non_existing_mp" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + partNumber := int32(4) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + PartNumber: &partNumber, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadObject_mp_success(s *S3Conf) error { + testName := "HeadObject_mp_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + partCount, partSize := 5, 1024 + partNumber := int32(3) + + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, err := uploadParts(s3client, partCount*partSize, partCount, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + PartNumber: &partNumber, + }) + cancel() + if err != nil { + return err + } + + if *out.ContentLength != int64(partSize) { + return fmt.Errorf("expected content length to be %v, instead got %v", partSize, *out.ContentLength) + } + if *out.ETag != *parts[partNumber-1].ETag { + return fmt.Errorf("expected ETag to be %v, instead got %v", *parts[partNumber-1].ETag, *out.ETag) + } + if *out.PartsCount != int32(partCount) { + return fmt.Errorf("expected part count to be %v, instead got %v", partCount, *out.PartsCount) + } + + return nil + }) +} + func HeadObject_success(s *S3Conf) error { testName := "HeadObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4356,7 +4434,7 @@ func UploadPartCopy_invalid_part_number(s *S3Conf) error { PartNumber: &partNumber, }) cancel() - if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPartNumber)); err != nil { return err }