Files
versitygw/tests/integration/HeadObject.go
niksis02 d2fa265fb8 feat: support sha512, md5, xxhash3, xxhash64, xxhash128 data integrity checksums
Integrate the new S3 checksum types in the gateway, including `SHA512`, `MD5`, `XXHASH64`, `XXHASH3`, and `XXHASH128`. This adds checksum calculation, validation, schema handling, and test coverage for the expanded checksum support.

These external packages have been used:
- `github.com/zeebo/xxh3` for `XXHASH3` and `XXHASH128`
- `github.com/cespare/xxhash/v2` for `XXHASH64`

Adjust integration tests because `aws-sdk-go-v2/service/s3` does not support automatic checksum calculation for the new checksum algorithms and returns an SDK-level error when only the checksum algorithm is provided. Only precalculated checksum values are acceptable for these checksum types.

References:
- `https://github.com/aws/aws-sdk-go-v2/issues/3404`
- `https://github.com/aws/aws-sdk-go-v2/issues/3403`
2026-05-04 08:50:39 -07:00

1462 lines
44 KiB
Go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package integration
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/s3err"
)
func HeadObject_non_existing_object(s *S3Conf) error {
testName := "HeadObject_non_existing_object"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: getPtr("my-obj"),
})
cancel()
if err := checkSdkApiErr(err, "NotFound"); err != nil {
return err
}
return nil
})
}
func HeadObject_invalid_part_number(s *S3Conf) error {
testName := "HeadObject_invalid_part_number"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
partNumber := int32(-3)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: getPtr("my-obj"),
PartNumber: &partNumber,
})
cancel()
if err := checkSdkApiErr(err, "BadRequest"); err != nil {
return err
}
return nil
})
}
func HeadObject_non_existing_dir_object(s *S3Conf) error {
testName := "HeadObject_non_existing_dir_object"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, dataLen := "my-obj", int64(1234567)
meta := map[string]string{
"key1": "val1",
"key2": "val2",
}
_, err := putObjectWithData(dataLen, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Metadata: meta,
}, s3client)
if err != nil {
return err
}
obj = "my-obj/"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
defer cancel()
if err := checkSdkApiErr(err, "NotFound"); err != nil {
return err
}
return nil
})
}
func HeadObject_directory_object_noslash(s *S3Conf) error {
testName := "HeadObject_directory_object_noslash"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj/"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
obj = "my-obj"
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err := checkSdkApiErr(err, "NotFound"); err != nil {
return err
}
return nil
})
}
const defaultContentType = "binary/octet-stream"
func HeadObject_not_enabled_checksum_mode(s *S3Conf) error {
testName := "HeadObject_not_enabled_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
_, err := putObjectWithData(500, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha1,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
})
}
func HeadObject_checksums(s *S3Conf) error {
testName := "HeadObject_checksums"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, checksumAlgo := range types.ChecksumAlgorithmCrc32.Values() {
key := fmt.Sprintf("obj-%v", i+1)
out, err := putObjectWithData(int64((i+1)*200), &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: checksumAlgo,
}, s3client, withPutObjectChecksumAlgo(checksumAlgo))
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
if err != nil {
return err
}
if res.ChecksumType != types.ChecksumTypeFullObject {
return fmt.Errorf("expected the %v object checksum type to be %v, instead got %v", key, types.ChecksumTypeFullObject, res.ChecksumType)
}
if got, expected := getString(getHeadObjectChecksum(res, checksumAlgo)), getString(getPutObjectChecksum(out.res, checksumAlgo)); got != expected {
return fmt.Errorf("expected %s checksum to be %v, instead got %v", checksumAlgo, expected, got)
}
}
return nil
})
}
func HeadObject_invalid_parent_dir(s *S3Conf) error {
testName := "HeadObject_invalid_parent_dir"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, dataLen := "not-a-dir", int64(1)
_, err := putObjectWithData(dataLen, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
obj = "not-a-dir/bad-obj"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
defer cancel()
if err := checkSdkApiErr(err, "NotFound"); err != nil {
return err
}
return nil
})
}
func HeadObject_with_range(s *S3Conf) error {
testName := "HeadObject_with_range"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, objLength := "my-obj", int64(100)
_, err := putObjectWithData(objLength, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
testRange := func(rg, contentRange string, cLength int64, expectErr bool) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
Range: &rg,
})
cancel()
if err == nil && expectErr {
return fmt.Errorf("%v: expected err 'RequestedRangeNotSatisfiable' error, instead got nil", rg)
}
if err != nil {
if !expectErr {
return err
}
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() != "RequestedRangeNotSatisfiable" {
return fmt.Errorf("%v: expected RequestedRangeNotSatisfiable, instead got %v", rg, ae.ErrorCode())
}
if ae.ErrorMessage() != "Requested Range Not Satisfiable" {
return fmt.Errorf("%v: expected the error message to be 'Requested Range Not Satisfiable', instead got %v", rg, ae.ErrorMessage())
}
return nil
}
return fmt.Errorf("%v: invalid error got %w", rg, err)
}
if getString(res.AcceptRanges) != "bytes" {
return fmt.Errorf("%v: expected accept ranges to be 'bytes', instead got %v", rg, getString(res.AcceptRanges))
}
if res.ContentLength == nil {
return fmt.Errorf("%v: expected non nil content-length", rg)
}
if *res.ContentLength != cLength {
return fmt.Errorf("%v: expected content-length to be %v, instead got %v", rg, cLength, *res.ContentLength)
}
if getString(res.ContentRange) != contentRange {
return fmt.Errorf("%v: expected content-range to be %v, instead got %v", rg, contentRange, getString(res.ContentRange))
}
return nil
}
// Reference server expectations for a 100-byte object.
for _, el := range []struct {
objRange string
contentRange string
contentLength int64
expectedErr bool
}{
// The following inputs should NOT produce an error and return the full object with empty Content-Range.
{"bytes=,", "", objLength, false},
{"bytes= -1", "", objLength, false},
{"bytes=--1", "", objLength, false},
{"bytes=0 -1", "", objLength, false},
{"bytes=0--1", "", objLength, false},
{"bytes=10-5", "", objLength, false}, // start > end treated as invalid
{"bytes=abc", "", objLength, false},
{"bytes=a-z", "", objLength, false},
{"foo=0-1", "", objLength, false}, // unsupported unit
{"bytes=00-01", "bytes 0-1/100", 2, false}, // valid numeric despite leading zeros
{"bytes=abc-xyz", "", objLength, false}, // retain legacy invalid pattern
{"bytes=100-x", "", objLength, false},
{"bytes=0-0,1-2", "", objLength, false}, // multiple ranges unsupported -> ignore
// Valid suffix ranges (negative forms)
{"bytes=-1", "bytes 99-99/100", 1, false},
{"bytes=-2", "bytes 98-99/100", 2, false},
{"bytes=-10", "bytes 90-99/100", 10, false},
{"bytes=-100", "bytes 0-99/100", objLength, false},
{"bytes=-101", "bytes 0-99/100", objLength, false}, // larger than object -> entire object
// Standard byte ranges
{"bytes=0-0", "bytes 0-0/100", 1, false},
{"bytes=0-99", "bytes 0-99/100", objLength, false},
{"bytes=0-100", "bytes 0-99/100", objLength, false}, // end past object -> trimmed
{"bytes=0-999999", "bytes 0-99/100", objLength, false},
{"bytes=1-99", "bytes 1-99/100", objLength - 1, false},
{"bytes=50-99", "bytes 50-99/100", 50, false},
{"bytes=50-", "bytes 50-99/100", 50, false},
{"bytes=0-", "bytes 0-99/100", objLength, false},
{"bytes=99-99", "bytes 99-99/100", 1, false},
// Ranges expected to produce RequestedRangeNotSatisfiable
{"bytes=-0", "", 0, true},
{"bytes=100-100", "", 0, true},
{"bytes=100-110", "", 0, true},
} {
if err := testRange(el.objRange, el.contentRange, el.contentLength, el.expectedErr); err != nil {
return err
}
}
return nil
})
}
func HeadObject_zero_len_with_range(s *S3Conf) error {
testName := "HeadObject_zero_len_with_range"
return headObject_zero_len_with_range_helper(testName, "my-obj", s)
}
func HeadObject_dir_with_range(s *S3Conf) error {
testName := "HeadObject_dir_with_range"
return headObject_zero_len_with_range_helper(testName, "my-dir/", s)
}
func HeadObject_conditional_reads(s *S3Conf) error {
testName := "HeadObject_conditional_reads"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
key := "my-obj"
obj, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
}, s3client)
if err != nil {
return err
}
errMod := getPtr("NotModified")
errCond := getPtr("PreconditionFailed")
// sleep one second to get dates before and after
// the object creation
time.Sleep(time.Second * 1)
before := time.Now().AddDate(0, 0, -3)
after := time.Now()
etag := obj.res.ETag
etagTrimmed := strings.Trim(*etag, `"`)
for i, test := range []struct {
ifmatch *string
ifnonematch *string
ifmodifiedsince *time.Time
ifunmodifiedsince *time.Time
err *string
}{
// all the cases when preconditions are either empty, true or false
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
{getPtr("invalid_etag"), etag, &before, &before, errCond},
{getPtr("invalid_etag"), etag, &before, &after, errCond},
{getPtr("invalid_etag"), etag, &before, nil, errCond},
{getPtr("invalid_etag"), etag, &after, &before, errCond},
{getPtr("invalid_etag"), etag, &after, &after, errCond},
{getPtr("invalid_etag"), etag, &after, nil, errCond},
{getPtr("invalid_etag"), etag, nil, &before, errCond},
{getPtr("invalid_etag"), etag, nil, &after, errCond},
{getPtr("invalid_etag"), etag, nil, nil, errCond},
{getPtr("invalid_etag"), nil, &before, &before, errCond},
{getPtr("invalid_etag"), nil, &before, &after, errCond},
{getPtr("invalid_etag"), nil, &before, nil, errCond},
{getPtr("invalid_etag"), nil, &after, &before, errCond},
{getPtr("invalid_etag"), nil, &after, &after, errCond},
{getPtr("invalid_etag"), nil, &after, nil, errCond},
{getPtr("invalid_etag"), nil, nil, &before, errCond},
{getPtr("invalid_etag"), nil, nil, &after, errCond},
{getPtr("invalid_etag"), nil, nil, nil, errCond},
{etag, getPtr("invalid_etag"), &before, &before, nil},
{etag, getPtr("invalid_etag"), &before, &after, nil},
{etag, getPtr("invalid_etag"), &before, nil, nil},
{etag, getPtr("invalid_etag"), &after, &before, nil},
{etag, getPtr("invalid_etag"), &after, &after, nil},
{etag, getPtr("invalid_etag"), &after, nil, nil},
{etag, getPtr("invalid_etag"), nil, &before, nil},
{etag, getPtr("invalid_etag"), nil, &after, nil},
{etag, getPtr("invalid_etag"), nil, nil, nil},
{etag, etag, &before, &before, errMod},
{etag, etag, &before, &after, errMod},
{etag, etag, &before, nil, errMod},
{etag, etag, &after, &before, errMod},
{etag, etag, &after, &after, errMod},
{etag, etag, &after, nil, errMod},
{etag, etag, nil, &before, errMod},
{etag, etag, nil, &after, errMod},
{etag, etag, nil, nil, errMod},
{etag, nil, &before, &before, nil},
{etag, nil, &before, &after, nil},
{etag, nil, &before, nil, nil},
{etag, nil, &after, &before, errMod},
{etag, nil, &after, &after, errMod},
{etag, nil, &after, nil, errMod},
{etag, nil, nil, &before, nil},
{etag, nil, nil, &after, nil},
{etag, nil, nil, nil, nil},
{nil, getPtr("invalid_etag"), &before, &before, errCond},
{nil, getPtr("invalid_etag"), &before, &after, nil},
{nil, getPtr("invalid_etag"), &before, nil, nil},
{nil, getPtr("invalid_etag"), &after, &before, errCond},
{nil, getPtr("invalid_etag"), &after, &after, nil},
{nil, getPtr("invalid_etag"), &after, nil, nil},
{nil, getPtr("invalid_etag"), nil, &before, errCond},
{nil, getPtr("invalid_etag"), nil, &after, nil},
{nil, getPtr("invalid_etag"), nil, nil, nil},
{nil, etag, &before, &before, errCond},
{nil, etag, &before, &after, errMod},
{nil, etag, &before, nil, errMod},
{nil, etag, &after, &before, errCond},
{nil, etag, &after, &after, errMod},
{nil, etag, &after, nil, errMod},
{nil, etag, nil, &before, errCond},
{nil, etag, nil, &after, errMod},
{nil, etag, nil, nil, errMod},
{nil, nil, &before, &before, errCond},
{nil, nil, &before, &after, nil},
{nil, nil, &before, nil, nil},
{nil, nil, &after, &before, errCond},
{nil, nil, &after, &after, errMod},
{nil, nil, &after, nil, errMod},
{nil, nil, nil, &before, errCond},
{nil, nil, nil, &after, nil},
{nil, nil, nil, nil, nil},
// if-match, if-non-match without quotes
{&etagTrimmed, getPtr("invalid_etag"), &before, &before, nil},
{&etagTrimmed, getPtr("invalid_etag"), &before, &after, nil},
{&etagTrimmed, getPtr("invalid_etag"), &before, nil, nil},
{&etagTrimmed, getPtr("invalid_etag"), &after, &before, nil},
{&etagTrimmed, getPtr("invalid_etag"), &after, &after, nil},
{&etagTrimmed, getPtr("invalid_etag"), &after, nil, nil},
{&etagTrimmed, getPtr("invalid_etag"), nil, &before, nil},
{&etagTrimmed, getPtr("invalid_etag"), nil, &after, nil},
{&etagTrimmed, getPtr("invalid_etag"), nil, nil, nil},
{&etagTrimmed, &etagTrimmed, &before, &before, errMod},
{&etagTrimmed, &etagTrimmed, &before, &after, errMod},
{&etagTrimmed, &etagTrimmed, &before, nil, errMod},
{&etagTrimmed, &etagTrimmed, &after, &before, errMod},
{&etagTrimmed, &etagTrimmed, &after, &after, errMod},
{&etagTrimmed, &etagTrimmed, &after, nil, errMod},
{&etagTrimmed, &etagTrimmed, nil, &before, errMod},
{&etagTrimmed, &etagTrimmed, nil, &after, errMod},
{&etagTrimmed, &etagTrimmed, nil, nil, errMod},
{&etagTrimmed, nil, &before, &before, nil},
{&etagTrimmed, nil, &before, &after, nil},
{&etagTrimmed, nil, &before, nil, nil},
{&etagTrimmed, nil, &after, &before, errMod},
{&etagTrimmed, nil, &after, &after, errMod},
{&etagTrimmed, nil, &after, nil, errMod},
{&etagTrimmed, nil, nil, &before, nil},
{&etagTrimmed, nil, nil, &after, nil},
{&etagTrimmed, nil, nil, nil, nil},
{nil, &etagTrimmed, &before, &before, errCond},
{nil, &etagTrimmed, &before, &after, errMod},
{nil, &etagTrimmed, &before, nil, errMod},
{nil, &etagTrimmed, &after, &before, errCond},
{nil, &etagTrimmed, &after, &after, errMod},
{nil, &etagTrimmed, &after, nil, errMod},
{nil, &etagTrimmed, nil, &before, errCond},
{nil, &etagTrimmed, nil, &after, errMod},
{nil, &etagTrimmed, nil, nil, errMod},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
IfMatch: test.ifmatch,
IfNoneMatch: test.ifnonematch,
IfModifiedSince: test.ifmodifiedsince,
IfUnmodifiedSince: test.ifunmodifiedsince,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
if err := checkSdkApiErr(err, *test.err); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func HeadObject_success(s *S3Conf) error {
testName := "HeadObject_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj, dataLen := "my-obj", int64(1234567)
meta := map[string]string{
"key1": "val1",
"key2": "val2",
}
ctype, cDisp, cEnc, cLang := defaultContentType, "cont-desp", "json", "eng"
cacheControl, expires := "cache-ctrl", time.Now().Add(time.Hour*2)
_, err := putObjectWithData(dataLen, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Metadata: meta,
ContentType: &ctype,
ContentDisposition: &cDisp,
ContentEncoding: &cEnc,
ContentLanguage: &cLang,
CacheControl: &cacheControl,
Expires: &expires,
Tagging: getPtr("key=value"),
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
defer cancel()
if err != nil {
return err
}
if !areMapsSame(out.Metadata, meta) {
return fmt.Errorf("incorrect object metadata")
}
contentLength := int64(0)
if out.ContentLength != nil {
contentLength = *out.ContentLength
}
if contentLength != dataLen {
return fmt.Errorf("expected data length %v, instead got %v",
dataLen, contentLength)
}
if getString(out.ContentType) != defaultContentType {
return fmt.Errorf("expected Content-Type %v, instead got %v",
defaultContentType, getString(out.ContentType))
}
if getString(out.ContentDisposition) != cDisp {
return fmt.Errorf("expected Content-Disposition %v, instead got %v",
cDisp, getString(out.ContentDisposition))
}
if getString(out.ContentEncoding) != cEnc {
return fmt.Errorf("expected Content-Encoding %v, instead got %v",
cEnc, getString(out.ContentEncoding))
}
if getString(out.ContentLanguage) != cLang {
return fmt.Errorf("expected Content-Language %v, instead got %v",
cLang, getString(out.ContentLanguage))
}
if getString(out.ExpiresString) != expires.UTC().Format(timefmt) {
return fmt.Errorf("expected Expiress %v, instead got %v",
expires.UTC().Format(timefmt), getString(out.ExpiresString))
}
if getString(out.CacheControl) != cacheControl {
return fmt.Errorf("expected Cache-Control %v, instead got %v",
cacheControl, getString(out.CacheControl))
}
if out.StorageClass != types.StorageClassStandard {
return fmt.Errorf("expected the storage class to be %v, instead got %v",
types.StorageClassStandard, out.StorageClass)
}
tagCount := int32(0)
if out.TagCount != nil {
tagCount = *out.TagCount
}
if tagCount != 1 {
return fmt.Errorf("expected the tagcount to be 1, instead got %v", tagCount)
}
return nil
})
}
func HeadObject_overrides_success(s *S3Conf) error {
testName := "HeadObject_overrides_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objKey := "test-object"
exp := time.Now()
_, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &objKey,
})
if err != nil {
return fmt.Errorf("failed to put object: %v", err)
}
for _, test := range []PublicBucketTestCase{
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseCacheControl: getPtr("max-age=90"),
})
return err
},
ExpectedErr: nil,
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentDisposition: getPtr("inline"),
})
return err
},
ExpectedErr: nil,
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentEncoding: getPtr("txt"),
})
return err
},
ExpectedErr: nil,
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentLanguage: getPtr("en"),
})
return err
},
ExpectedErr: nil,
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentType: getPtr("application/json"),
})
return err
},
ExpectedErr: nil,
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseExpires: &exp,
})
return err
},
ExpectedErr: nil,
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
err := test.Call(ctx)
cancel()
if err == nil && test.ExpectedErr != nil {
return fmt.Errorf("%v: expected err %v, instead got successful response", test.Action, test.ExpectedErr)
}
if err != nil {
if test.ExpectedErr == nil {
return fmt.Errorf("%v: expected no error, instead got %v", test.Action, err)
}
apiErr, ok := test.ExpectedErr.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type provided in the test, expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return err
}
}
}
return nil
})
}
func HeadObject_overrides_presign_success(s *S3Conf) error {
testName := "HeadObject_overrides_presign_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objKey := "test-object"
_, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &objKey,
})
if err != nil {
return fmt.Errorf("failed to put object: %v", err)
}
testCases := []struct {
name string
queryParam string
expectedHeader string
expectedValue string
}{
{
name: "response-cache-control",
queryParam: "response-cache-control=no-cache",
expectedHeader: "Cache-Control",
expectedValue: "no-cache",
},
{
name: "response-content-disposition",
queryParam: "response-content-disposition=attachment%3B%20filename%3D%22test.txt%22",
expectedHeader: "Content-Disposition",
expectedValue: "attachment; filename=\"test.txt\"",
},
{
name: "response-content-encoding",
queryParam: "response-content-encoding=txt",
expectedHeader: "Content-Encoding",
expectedValue: "txt",
},
{
name: "response-content-language",
queryParam: "response-content-language=en-US",
expectedHeader: "Content-Language",
expectedValue: "en-US",
},
{
name: "response-content-type",
queryParam: "response-content-type=text%2Fplain",
expectedHeader: "Content-Type",
expectedValue: "text/plain",
},
{
name: "response-expires",
queryParam: "response-expires=Thu%2C%2001%20Dec%202024%2016%3A00%3A00%20GMT",
expectedHeader: "Expires",
expectedValue: "Thu, 01 Dec 2024 16:00:00 GMT",
},
}
for _, tc := range testCases {
req, err := createSignedReq(
http.MethodHead,
s.endpoint,
fmt.Sprintf("%s/%s?%s", bucket, objKey, tc.queryParam),
s.awsID,
s.awsSecret,
"s3",
s.awsRegion,
nil,
time.Now(),
nil,
)
if err != nil {
return fmt.Errorf("failed to create signed request for %s: %v", tc.name, err)
}
resp, err := s.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to execute request for %s: %v", tc.name, err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected status 200 for %s, got %d", tc.name, resp.StatusCode)
}
actualValue := resp.Header.Get(tc.expectedHeader)
if actualValue != tc.expectedValue {
return fmt.Errorf("expected %s header to be %q for %s, got %q",
tc.expectedHeader, tc.expectedValue, tc.name, actualValue)
}
}
// Test multiple override parameters together
multiParam := "response-cache-control=max-age%3D3600&response-content-type=application%2Fjson&response-content-disposition=inline"
req, err := createSignedReq(
http.MethodHead,
s.endpoint,
fmt.Sprintf("%s/%s?%s", bucket, objKey, multiParam),
s.awsID,
s.awsSecret,
"s3",
s.awsRegion,
nil,
time.Now(),
nil,
)
if err != nil {
return fmt.Errorf("failed to create signed request for multiple overrides: %v", err)
}
resp, err := s.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to execute request for multiple overrides: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("expected status 200 for multiple overrides, got %d", resp.StatusCode)
}
expectedHeaders := map[string]string{
"Cache-Control": "max-age=3600",
"Content-Type": "application/json",
"Content-Disposition": "inline",
}
for headerName, expectedValue := range expectedHeaders {
actualValue := resp.Header.Get(headerName)
if actualValue != expectedValue {
return fmt.Errorf("expected %s header to be %q for multiple overrides, got %q",
headerName, expectedValue, actualValue)
}
}
return nil
})
}
func HeadObject_range_and_part_number(s *S3Conf) error {
testName := "HeadObject_range_and_part_number"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
_, err := putObjectWithData(100, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
pn := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
Range: getPtr("bytes=0-9"),
PartNumber: &pn,
})
cancel()
return checkSdkApiErr(err, "BadRequest")
})
}
func HeadObject_mp_part_number_exceeds_parts_count(s *S3Conf) error {
testName := "HeadObject_mp_part_number_exceeds_parts_count"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
out, err := createMp(s3client, bucket, obj)
if err != nil {
return err
}
const partCount = int64(5)
parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId)
if err != nil {
return err
}
compParts := make([]types.CompletedPart, len(parts))
for i, p := range parts {
compParts[i] = types.CompletedPart{
ETag: p.ETag,
PartNumber: p.PartNumber,
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: out.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
// partNumber exceeds the number of parts in the completed upload
pn := int32(partCount + 1)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &pn,
})
cancel()
return checkSdkApiErr(err, "RequestedRangeNotSatisfiable")
})
}
func HeadObject_mp_part_number_success(s *S3Conf) error {
testName := "HeadObject_mp_part_number_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
out, err := createMp(s3client, bucket, obj)
if err != nil {
return err
}
const partCount = int64(3)
const partSize = int64(5 * 1024 * 1024)
const totalSize = partCount * partSize
parts, _, err := uploadParts(s3client, totalSize, partCount, bucket, obj, *out.UploadId)
if err != nil {
return err
}
compParts := make([]types.CompletedPart, len(parts))
for i, p := range parts {
compParts[i] = types.CompletedPart{
ETag: p.ETag,
PartNumber: p.PartNumber,
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: out.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
for i := range partCount {
pn := int32(i + 1)
startByte := i * partSize
endByte := startByte + partSize - 1
expectedContentRange := fmt.Sprintf("bytes %d-%d/%d", startByte, endByte, totalSize)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &pn,
})
cancel()
if err != nil {
return fmt.Errorf("part %d: %w", pn, err)
}
if res.PartsCount == nil {
return fmt.Errorf("part %d: expected non-nil x-amz-mp-parts-count", pn)
}
if *res.PartsCount != int32(partCount) {
return fmt.Errorf("part %d: expected PartsCount %d, got %d", pn, partCount, *res.PartsCount)
}
if getString(res.ContentRange) != expectedContentRange {
return fmt.Errorf("part %d: expected Content-Range %q, got %q", pn, expectedContentRange, getString(res.ContentRange))
}
if res.ContentLength == nil || *res.ContentLength != partSize {
return fmt.Errorf("part %d: expected Content-Length %d, got %v", pn, partSize, res.ContentLength)
}
if getString(res.AcceptRanges) != "bytes" {
return fmt.Errorf("part %d: expected Accept-Ranges 'bytes', got %q", pn, getString(res.AcceptRanges))
}
}
return nil
})
}
func HeadObject_non_mp_part_number_1_success(s *S3Conf) error {
testName := "HeadObject_non_mp_part_number_1_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "put-object-part1"
const objSize = int64(1234)
_, err := putObjectWithData(objSize, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
pn := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &pn,
})
cancel()
if err != nil {
return err
}
if res.ContentLength == nil || *res.ContentLength != objSize {
return fmt.Errorf("expected ContentLength %d, got %v", objSize, res.ContentLength)
}
expectedCRange := fmt.Sprintf("bytes 0-%d/%d", objSize-1, objSize)
if getString(res.ContentRange) != expectedCRange {
return fmt.Errorf("expected Content-Range to be %s, instead got %s", expectedCRange, getString(res.ContentRange))
}
if res.PartsCount != nil {
return fmt.Errorf("expected nil PartsCount for non-multipart object, got %d", *res.PartsCount)
}
if getString(res.AcceptRanges) != "bytes" {
return fmt.Errorf("expected Accept-Ranges 'bytes', got %q", getString(res.AcceptRanges))
}
return nil
})
}
func HeadObject_by_range_resp_status(s *S3Conf) error {
testName := "HeadObject_by_range_resp_status"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
objLength := int64(100)
_, err := putObjectWithData(objLength, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
}, s3client)
if err != nil {
return err
}
checkRangeStatus := func(rng string, expectedStatus int) error {
req, err := createSignedReq(
http.MethodHead,
s.endpoint,
fmt.Sprintf("%v/%v", bucket, obj),
s.awsID,
s.awsSecret,
"s3",
s.awsRegion,
nil,
time.Now(),
map[string]string{"Range": rng},
)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != expectedStatus {
return fmt.Errorf("range %q: expected status %d, instead got %d", rng, expectedStatus, resp.StatusCode)
}
return nil
}
for _, tc := range []struct {
rng string
expectedStatus int
}{
// Invalid/ignored ranges → full object, no Content-Range → 200
{"bytes=,", http.StatusOK},
{"bytes= -1", http.StatusOK},
{"bytes=--1", http.StatusOK},
{"bytes=0 -1", http.StatusOK},
{"bytes=0--1", http.StatusOK},
{"bytes=10-5", http.StatusOK},
{"bytes=abc", http.StatusOK},
{"bytes=a-z", http.StatusOK},
{"foo=0-1", http.StatusOK},
{"bytes=abc-xyz", http.StatusOK},
{"bytes=100-x", http.StatusOK},
{"bytes=0-0,1-2", http.StatusOK},
{fmt.Sprintf("bytes=%v-%v", objLength+2, objLength-100), http.StatusOK},
// Valid ranges → partial content, non-empty Content-Range → 206
{"bytes=00-01", http.StatusPartialContent},
{"bytes=-1", http.StatusPartialContent},
{"bytes=-2", http.StatusPartialContent},
{"bytes=-10", http.StatusPartialContent},
{"bytes=-100", http.StatusPartialContent},
{"bytes=-101", http.StatusPartialContent},
{"bytes=0-0", http.StatusPartialContent},
{"bytes=0-99", http.StatusPartialContent},
{"bytes=0-100", http.StatusPartialContent},
{"bytes=0-999999", http.StatusPartialContent},
{"bytes=1-99", http.StatusPartialContent},
{"bytes=50-99", http.StatusPartialContent},
{"bytes=50-", http.StatusPartialContent},
{"bytes=0-", http.StatusPartialContent},
{"bytes=99-99", http.StatusPartialContent},
} {
if err := checkRangeStatus(tc.rng, tc.expectedStatus); err != nil {
return err
}
}
return nil
})
}
func HeadObject_empty_object_part_number_1(s *S3Conf) error {
testName := "HeadObject_empty_object_part_number_1"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "empty-obj"
out, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
}, s3client)
if err != nil {
return err
}
pn := int32(1)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
PartNumber: &pn,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
if err != nil {
return err
}
if res.ContentRange != nil {
return fmt.Errorf("expected nil Content-Range for empty object with partNumber=1, got %q", *res.ContentRange)
}
if res.ContentLength == nil || *res.ContentLength != 0 {
return fmt.Errorf("expected ContentLength 0, got %v", res.ContentLength)
}
if getString(res.ChecksumSHA256) != getString(out.res.ChecksumSHA256) {
return fmt.Errorf("expected sha256 checksum %v, got %v",
getString(out.res.ChecksumSHA256), getString(res.ChecksumSHA256))
}
return nil
})
}
func HeadObject_mp_part_number_resp_status(s *S3Conf) error {
testName := "HeadObject_mp_part_number_resp_status"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
out, err := createMp(s3client, bucket, obj)
if err != nil {
return err
}
const partCount = int64(2)
parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, obj, *out.UploadId)
if err != nil {
return err
}
compParts := make([]types.CompletedPart, len(parts))
for i, p := range parts {
compParts[i] = types.CompletedPart{
ETag: p.ETag,
PartNumber: p.PartNumber,
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: out.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
req, err := createSignedReq(
http.MethodHead,
s.endpoint,
fmt.Sprintf("%v/%v?partNumber=1", bucket, obj),
s.awsID,
s.awsSecret,
"s3",
s.awsRegion,
nil,
time.Now(),
nil,
)
if err != nil {
return err
}
resp, err := s.httpClient.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
return fmt.Errorf("expected response status to be %v, instead got %v",
http.StatusPartialContent, resp.StatusCode)
}
return nil
})
}
func HeadObject_ranged_with_checksum_mode(s *S3Conf) error {
testName := "HeadObject_ranged_with_checksum_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
checkNoChecksums := func(res *s3.HeadObjectOutput) error {
for _, algo := range types.ChecksumAlgorithmCrc32.Values() {
if checksum := getHeadObjectChecksum(res, algo); checksum != nil {
return fmt.Errorf("expected nil %s checksum, instead got %v", algo, *checksum)
}
}
return nil
}
// Sub-test 1: regular object with Range header and ChecksumMode enabled
regularObj := "regular-obj"
_, err := putObjectWithData(500, &s3.PutObjectInput{
Bucket: &bucket,
Key: &regularObj,
ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
}, s3client)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &regularObj,
Range: getPtr("bytes=0-99"),
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
if err != nil {
return fmt.Errorf("ranged HEAD on regular object: %w", err)
}
if err := checkNoChecksums(res); err != nil {
return fmt.Errorf("ranged HEAD on regular object: %w", err)
}
// Sub-test 2: multipart object with partNumber and ChecksumMode enabled
mpObj := "mp-obj"
mpOut, err := createMp(s3client, bucket, mpObj)
if err != nil {
return err
}
const partCount = int64(2)
parts, _, err := uploadParts(s3client, partCount*5*1024*1024, partCount, bucket, mpObj, *mpOut.UploadId)
if err != nil {
return err
}
compParts := make([]types.CompletedPart, len(parts))
for i, p := range parts {
compParts[i] = types.CompletedPart{
ETag: p.ETag,
PartNumber: p.PartNumber,
}
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &mpObj,
UploadId: mpOut.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
pn := int32(1)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
mpRes, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &mpObj,
PartNumber: &pn,
ChecksumMode: types.ChecksumModeEnabled,
})
cancel()
if err != nil {
return fmt.Errorf("partNumber HEAD on MP object: %w", err)
}
if err := checkNoChecksums(mpRes); err != nil {
return fmt.Errorf("partNumber HEAD on MP object: %w", err)
}
return nil
})
}
func HeadObject_overrides_fail_public(s *S3Conf) error {
testName := "HeadObject_overrides_fail_public"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
rootClient := s.GetClient()
err := grantPublicBucketPolicy(rootClient, bucket, policyTypeObject)
if err != nil {
return err
}
objKey := "test-object"
exp := time.Now()
_, err = rootClient.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &objKey,
})
if err != nil {
return fmt.Errorf("failed to put object: %v", err)
}
for _, test := range []PublicBucketTestCase{
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseCacheControl: getPtr("max-age=90"),
})
return err
},
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentDisposition: getPtr("inline"),
})
return err
},
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentEncoding: getPtr("txt"),
})
return err
},
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentLanguage: getPtr("en"),
})
return err
},
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseContentType: getPtr("application/json"),
})
return err
},
},
{
Action: "HeadObject",
Call: func(ctx context.Context) error {
_, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &objKey,
ResponseExpires: &exp,
})
return err
},
},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
err := test.Call(ctx)
cancel()
if err == nil {
return fmt.Errorf("%v: expected a BadRequest error, instead got successful response", test.Action)
}
if err := checkSdkApiErr(err, "BadRequest"); err != nil {
return err
}
}
return nil
}, withAnonymousClient())
}