mirrors/versitygw
1
0
Fork 0
mirror of https://github.com/versity/versitygw.git synced 2026-07-20 23:12:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Integrates object integrity checksums(CRC32, CRC32C, SHA1, SHA256) into the gateway

Browse Source
This commit is contained in:
niksis02
2025-01-13 22:28:46 +04:00
parent f42c20297b
commit 6956757557
17 changed files with 3072 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/azure/azure.go
+14 -12
View File
@@ -905,9 +905,9 @@ func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMult
}
// Each part is translated into an uncommitted block in a newly created blob in staging area
func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
return "", err
return nil, err
}
// TODO: request streamable version of StageBlock()
@@ -916,32 +916,34 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta
// the body in memory to create an io.ReadSeekCloser
rdr, err := getReadSeekCloser(input.Body)
if err != nil {
return "", err
return nil, err
}
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return "", err
return nil, err
}
// block id serves as etag here
etag = blockIDInt32ToBase64(*input.PartNumber)
etag := blockIDInt32ToBase64(*input.PartNumber)
_, err = client.StageBlock(ctx, etag, rdr, nil)
if err != nil {
return "", parseMpError(err)
return nil, parseMpError(err)
}
return etag, nil
return &s3.UploadPartOutput{
ETag: &etag,
}, nil
}
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyObjectResult{}, nil
return s3response.CopyPartResult{}, nil
}
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
return s3response.CopyObjectResult{}, err
return s3response.CopyPartResult{}, err
}
eTag := blockIDInt32ToBase64(*input.PartNumber)
@@ -949,10 +951,10 @@ func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInp
//TODO: the action returns not implemented on azurite, maybe in production this will work?
_, err = client.StageBlockFromURL(ctx, eTag, *input.CopySource, nil)
if err != nil {
return s3response.CopyObjectResult{}, parseMpError(err)
return s3response.CopyPartResult{}, parseMpError(err)
}
return s3response.CopyObjectResult{}, nil
return s3response.CopyPartResult{}, nil
}
// Lists all uncommitted parts from the blob
backend/backend.go
+6 -6
View File
@@ -53,8 +53,8 @@ type Backend interface {
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error)
@@ -166,11 +166,11 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
backend/posix/posix.go
+727 -78
View File
File diff suppressed because it is too large Load Diff
backend/s3proxy/s3.go
+6 -10
View File
@@ -332,28 +332,24 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
}, nil
}
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
if err != nil {
return "", handleError(err)
}
return *output.ETag, nil
return output, handleError(err)
}
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
output, err := s.client.UploadPartCopy(ctx, input)
if err != nil {
return s3response.CopyObjectResult{}, handleError(err)
return s3response.CopyPartResult{}, handleError(err)
}
return s3response.CopyObjectResult{
return s3response.CopyPartResult{
LastModified: *output.CopyPartResult.LastModified,
ETag: *output.CopyPartResult.ETag,
ETag: output.CopyPartResult.ETag,
}, nil
}
s3api/controllers/backend_moq_test.go
+6 -6
View File
@@ -170,10 +170,10 @@ var _ backend.Backend = &BackendMock{}
// StringFunc: func() string {
// panic("mock out the String method")
// },
// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
// UploadPartFunc: func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
// panic("mock out the UploadPart method")
// },
// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
// UploadPartCopyFunc: func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
// panic("mock out the UploadPartCopy method")
// },
// }
@@ -331,10 +331,10 @@ type BackendMock struct {
StringFunc func() string
// UploadPartFunc mocks the UploadPart method.
UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error)
UploadPartFunc func(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error)
// UploadPartCopyFunc mocks the UploadPartCopy method.
UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
UploadPartCopyFunc func(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
// calls tracks calls to the methods.
calls struct {
@@ -2620,7 +2620,7 @@ func (mock *BackendMock) StringCalls() []struct {
}
// UploadPart calls UploadPartFunc.
func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (string, error) {
func (mock *BackendMock) UploadPart(contextMoqParam context.Context, uploadPartInput *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
if mock.UploadPartFunc == nil {
panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called")
}
@@ -2656,7 +2656,7 @@ func (mock *BackendMock) UploadPartCalls() []struct {
}
// UploadPartCopy calls UploadPartCopyFunc.
func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
func (mock *BackendMock) UploadPartCopy(contextMoqParam context.Context, uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
if mock.UploadPartCopyFunc == nil {
panic("BackendMock.UploadPartCopyFunc: method is nil but Backend.UploadPartCopy was just called")
}
s3api/controllers/base.go
+244 -16
View File
@@ -485,12 +485,27 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
})
}
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
if c.debug {
log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
}
return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionGetObject,
BucketOwner: parsedAcl.Owner,
})
}
ctx.Locals("logResBody", false)
res, err := c.be.GetObject(ctx.Context(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
Range: &acceptRange,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
if res != nil {
@@ -568,6 +583,30 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
})
}
if res.ChecksumCRC32 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
}
if res.ChecksumCRC32C != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
}
if res.ChecksumSHA1 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
}
if res.ChecksumSHA256 != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
@@ -2007,6 +2046,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
})
}
algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
@@ -2016,16 +2066,55 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
ctx.Locals("logReqBody", false)
etag, err := c.be.UploadPart(ctx.Context(),
res, err := c.be.UploadPart(ctx.Context(),
&s3.UploadPartInput{
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: &partNumber,
ContentLength: &contentLength,
Body: body,
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: &partNumber,
ContentLength: &contentLength,
Body: body,
ChecksumAlgorithm: algorithm,
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
})
ctx.Response().Header.Set("Etag", etag)
if err == nil {
headers := []utils.CustomHeader{}
if res.ETag != nil {
headers = append(headers, utils.CustomHeader{
Key: "ETag",
Value: *res.ETag,
})
}
if res.ChecksumCRC32 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
}
if res.ChecksumCRC32C != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
}
if res.ChecksumSHA1 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
}
if res.ChecksumSHA256 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
utils.SetResponseHeaders(ctx, headers)
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
@@ -2256,6 +2345,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
metaDirective = types.MetadataDirectiveReplace
}
checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
if err != nil {
if c.debug {
log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCopyObject,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CopyObject(ctx.Context(),
&s3.CopyObjectInput{
Bucket: &bucket,
@@ -2269,6 +2373,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Metadata: metadata,
MetadataDirective: metaDirective,
StorageClass: types.StorageClass(storageClass),
ChecksumAlgorithm: checksumAlgorithm,
})
if err == nil {
hdrs := []utils.CustomHeader{}
@@ -2368,6 +2473,17 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
})
}
algorithm, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionPutObject,
BucketOwner: parsedAcl.Owner,
})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
@@ -2390,6 +2506,11 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ObjectLockRetainUntilDate: &objLock.RetainUntilDate,
ObjectLockMode: objLock.ObjectLockMode,
ObjectLockLegalHoldStatus: objLock.LegalHoldStatus,
ChecksumAlgorithm: algorithm,
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
})
if err != nil {
return SendResponse(ctx, err,
@@ -2417,6 +2538,30 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: res.VersionID,
})
}
if getstring(res.ChecksumCRC32) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: getstring(res.ChecksumCRC32),
})
}
if getstring(res.ChecksumCRC32C) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: getstring(res.ChecksumCRC32C),
})
}
if getstring(res.ChecksumSHA1) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: getstring(res.ChecksumSHA1),
})
}
if getstring(res.ChecksumSHA256) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: getstring(res.ChecksumSHA256),
})
}
utils.SetResponseHeaders(ctx, hdrs)
@@ -2934,12 +3079,27 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
})
}
checksumMode := types.ChecksumMode(ctx.Get("x-amz-checksum-mode"))
if checksumMode != "" && checksumMode != types.ChecksumModeEnabled {
if c.debug {
log.Printf("invalid x-amz-checksum-mode header value: %v\n", checksumMode)
}
return SendResponse(ctx, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-mode"),
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionHeadObject,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.HeadObject(ctx.Context(),
&s3.HeadObjectInput{
Bucket: &bucket,
Key: &key,
PartNumber: partNumber,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
PartNumber: partNumber,
VersionId: &versionId,
ChecksumMode: checksumMode,
})
if err != nil {
if res != nil {
@@ -3022,6 +3182,30 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: string(res.StorageClass),
})
}
if res.ChecksumCRC32 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32",
Value: *res.ChecksumCRC32,
})
}
if res.ChecksumCRC32C != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc32c",
Value: *res.ChecksumCRC32C,
})
}
if res.ChecksumSHA1 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha1",
Value: *res.ChecksumSHA1,
})
}
if res.ChecksumSHA256 != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-sha256",
Value: *res.ChecksumSHA256,
})
}
contentType := getstring(res.ContentType)
if contentType == "" {
@@ -3232,6 +3416,20 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
})
}
_, checksums, err := utils.ParseChecksumHeaders(ctx)
if err != nil {
if c.debug {
log.Printf("err parsing checksum headers: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCompleteMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -3240,6 +3438,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
MultipartUpload: &types.CompletedMultipartUpload{
Parts: data.Parts,
},
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
})
if err == nil {
if getstring(res.VersionId) != "" {
@@ -3305,6 +3507,21 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
metadata := utils.GetUserMetaData(&ctx.Request().Header)
checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
if err != nil {
if c.debug {
log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCreateMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}
res, err := c.be.CreateMultipartUpload(ctx.Context(),
&s3.CreateMultipartUploadInput{
Bucket: &bucket,
@@ -3316,7 +3533,18 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
ObjectLockMode: objLockState.ObjectLockMode,
ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
Metadata: metadata,
ChecksumAlgorithm: checksumAlgorithm,
})
if err == nil {
if checksumAlgorithm != "" {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-checksum-algorithm",
Value: string(checksumAlgorithm),
},
})
}
}
return SendXMLResponse(ctx, res, err,
&MetaOpts{
Logger: c.logger,
s3api/controllers/base_test.go
+88 -4
View File
@@ -232,6 +232,9 @@ func TestS3ApiController_GetActions(t *testing.T) {
getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -329,6 +332,15 @@ func TestS3ApiController_GetActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-invalid-checksum-mode",
app: app,
args: args{
req: invalidChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-get-object-success",
app: app,
@@ -971,14 +983,14 @@ func TestS3ApiController_PutActions(t *testing.T) {
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, nil
},
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
return "hello", nil
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return &s3.UploadPartOutput{}, nil
},
PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
return nil
},
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, nil
},
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
return nil
@@ -1012,6 +1024,11 @@ func TestS3ApiController_PutActions(t *testing.T) {
cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
// CopyObject invalid checksum algorithm
cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObjectAcl success
aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1033,6 +1050,40 @@ func TestS3ApiController_PutActions(t *testing.T) {
invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
// PutObject invalid checksum algorithm
invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObject invalid base64 checksum
invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
// PutObject invalid crc32
invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
// PutObject invalid crc32c
invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
// PutObject invalid sha1
invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
// PutObject invalid sha256
invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
// PutObject multiple checksum headers
mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
// PutObject checksum algorithm and header mismatch
checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
tests := []struct {
name string
app *fiber.App
@@ -1175,6 +1226,15 @@ func TestS3ApiController_PutActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Copy-object-invalid-checksum-algorithm",
app: app,
args: args{
req: cpyInvChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Copy-object-success",
app: app,
@@ -1642,6 +1702,9 @@ func TestS3ApiController_HeadObject(t *testing.T) {
})
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -1658,6 +1721,15 @@ func TestS3ApiController_HeadObject(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Head-object-invalid-checksum-mode",
app: app,
args: args{
req: invChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Head-object-error",
app: appErr,
@@ -1735,6 +1807,9 @@ func TestS3ApiController_CreateActions(t *testing.T) {
})
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
tests := []struct {
name string
app *fiber.App
@@ -1796,6 +1871,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Create-multipart-upload-invalid-checksum-algorithm",
app: app,
args: args{
req: invChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Create-multipart-upload-success",
app: app,
s3api/middlewares/md5.go
+1 -1
View File
@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
}
sum := md5.Sum(ctx.Body())
calculatedSum := utils.Md5SumString(sum[:])
calculatedSum := utils.Base64SumString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
s3api/utils/auth-reader.go
+1 -1
View File
@@ -56,7 +56,7 @@ func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, de
var hr *HashReader
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hr, _ = NewHashReader(r, "", HashTypeSha256)
hr, _ = NewHashReader(r, "", HashTypeSha256Hex)
} else {
hr, _ = NewHashReader(r, "", HashTypeNone)
}
s3api/utils/csum-reader.go
+65 -10
View File
@@ -16,13 +16,16 @@ package utils
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"hash"
"hash/crc32"
"io"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
@@ -31,11 +34,19 @@ type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
HashTypeMd5 = "md5"
// HashTypeSha256 generates SHA256 checksum for the data stream
HashTypeSha256 = "sha256"
HashTypeMd5 HashType = "md5"
// HashTypeSha256 generates SHA256 Base64-Encoded checksum for the data stream
HashTypeSha256 HashType = "sha256"
// HashTypeSha256Hex generates SHA256 hex encoded checksum for the data stream
HashTypeSha256Hex HashType = "sha256-hex"
// HashTypeSha1 generates SHA1 Base64-Encoded checksum for the data stream
HashTypeSha1 HashType = "sha1"
// HashTypeCRC32 generates CRC32 Base64-Encoded checksum for the data stream
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone = "none"
HashTypeNone HashType = "none"
)
// HashReader is an io.Reader that calculates the checksum
@@ -62,8 +73,16 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
switch ht {
case HashTypeMd5:
hash = md5.New()
case HashTypeSha256Hex:
hash = sha256.New()
case HashTypeSha256:
hash = sha256.New()
case HashTypeSha1:
hash = sha1.New()
case HashTypeCRC32:
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeNone:
hash = noop{}
default:
@@ -88,15 +107,35 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
}
case HashTypeSha256:
sum := hex.EncodeToString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
case HashTypeCRC32:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32)
}
case HashTypeCRC32C:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc32c)
}
case HashTypeSha1:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha1)
}
case HashTypeSha256:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
default:
return n, errInvalidHashType
}
@@ -104,20 +143,36 @@ func (hr *HashReader) Read(p []byte) (int, error) {
return n, readerr
}
func (hr *HashReader) SetReader(r io.Reader) {
hr.r = r
}
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
return Md5SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256Hex:
return hex.EncodeToString(hr.hash.Sum(nil))
case HashTypeCRC32:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC32C:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha1:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
}
func (hr *HashReader) Type() HashType {
return hr.hashType
}
// Md5SumString converts the hash bytes to the string checksum value
func Md5SumString(b []byte) string {
func Base64SumString(b []byte) string {
return base64.StdEncoding.EncodeToString(b)
}
s3api/utils/utils.go
+75 -1
View File
@@ -16,6 +16,7 @@ package utils
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -296,7 +297,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
if _, ok := attrs[s3response.ObjectAttributesStorageClass]; !ok {
output.StorageClass = ""
}
fmt.Printf("%+v\n", output)
if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
output.Checksum = nil
}
return output
}
@@ -456,3 +459,74 @@ func shouldEscape(c byte) bool {
return true
}
func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) {
sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-sdk-checksum-algorithm"))
err := IsChecksumAlgorithmValid(sdkAlgorithm)
if err != nil {
return "", nil, err
}
checksums := map[types.ChecksumAlgorithm]string{
types.ChecksumAlgorithmCrc32: ctx.Get("x-amz-checksum-crc32"),
types.ChecksumAlgorithmCrc32c: ctx.Get("x-amz-checksum-crc32c"),
types.ChecksumAlgorithmSha1: ctx.Get("x-amz-checksum-sha1"),
types.ChecksumAlgorithmSha256: ctx.Get("x-amz-checksum-sha256"),
}
headerCtr := 0
for al, val := range checksums {
if val != "" && !IsValidChecksum(val, al) {
return sdkAlgorithm, checksums, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(al))))
}
// If any other checksum value is provided,
// rather than x-amz-sdk-checksum-algorithm
if sdkAlgorithm != "" && sdkAlgorithm != al && val != "" {
return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
if val != "" {
headerCtr++
}
if headerCtr > 1 {
return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)
}
}
return sdkAlgorithm, checksums, nil
}
var checksumLengths = map[types.ChecksumAlgorithm]int{
types.ChecksumAlgorithmCrc32: 4,
types.ChecksumAlgorithmCrc32c: 4,
types.ChecksumAlgorithmSha1: 20,
types.ChecksumAlgorithmSha256: 32,
}
func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool {
decoded, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {
return false
}
expectedLength, exists := checksumLengths[algorithm]
if !exists {
return false
}
return len(decoded) == expectedLength
}
func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error {
if alg != "" &&
alg != types.ChecksumAlgorithmCrc32 &&
alg != types.ChecksumAlgorithmCrc32c &&
alg != types.ChecksumAlgorithmSha1 &&
alg != types.ChecksumAlgorithmSha256 {
return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)
}
return nil
}
s3api/utils/utils_test.go
+153
View File
@@ -527,3 +527,156 @@ func Test_escapePath(t *testing.T) {
})
}
}
func TestIsChecksumAlgorithmValid(t *testing.T) {
type args struct {
alg types.ChecksumAlgorithm
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "empty",
args: args{
alg: "",
},
wantErr: false,
},
{
name: "crc32",
args: args{
alg: types.ChecksumAlgorithmCrc32,
},
wantErr: false,
},
{
name: "crc32c",
args: args{
alg: types.ChecksumAlgorithmCrc32c,
},
wantErr: false,
},
{
name: "sha1",
args: args{
alg: types.ChecksumAlgorithmSha1,
},
wantErr: false,
},
{
name: "sha256",
args: args{
alg: types.ChecksumAlgorithmSha256,
},
wantErr: false,
},
{
name: "invalid",
args: args{
alg: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := IsChecksumAlgorithmValid(tt.args.alg); (err != nil) != tt.wantErr {
t.Errorf("IsChecksumAlgorithmValid() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestIsValidChecksum(t *testing.T) {
type args struct {
checksum string
algorithm types.ChecksumAlgorithm
}
tests := []struct {
name string
args args
want bool
}{
{
name: "invalid-base64",
args: args{
checksum: "invalid_base64_string",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: false,
},
{
name: "invalid-crc32",
args: args{
checksum: "YXNkZmFzZGZhc2Rm",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: false,
},
{
name: "valid-crc32",
args: args{
checksum: "ww2FVQ==",
algorithm: types.ChecksumAlgorithmCrc32,
},
want: true,
},
{
name: "invalid-crc32c",
args: args{
checksum: "Zmdoa2doZmtnZmhr",
algorithm: types.ChecksumAlgorithmCrc32c,
},
want: false,
},
{
name: "valid-crc32c",
args: args{
checksum: "DOsb4w==",
algorithm: types.ChecksumAlgorithmCrc32c,
},
want: true,
},
{
name: "invalid-sha1",
args: args{
checksum: "YXNkZmFzZGZhc2RmYXNkZnNhZGZzYWRm",
algorithm: types.ChecksumAlgorithmSha1,
},
want: false,
},
{
name: "valid-sha1",
args: args{
checksum: "L4q6V59Zcwn12wyLIytoE2c1ugk=",
algorithm: types.ChecksumAlgorithmSha1,
},
want: true,
},
{
name: "invalid-sha256",
args: args{
checksum: "Zmdoa2doZmtnZmhrYXNkZmFzZGZhc2RmZHNmYXNkZg==",
algorithm: types.ChecksumAlgorithmSha256,
},
want: false,
},
{
name: "valid-sha256",
args: args{
checksum: "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=",
algorithm: types.ChecksumAlgorithmSha256,
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidChecksum(tt.args.checksum, tt.args.algorithm); got != tt.want {
t.Errorf("IsValidChecksum() = %v, want %v", got, tt.want)
}
})
}
}
s3err/s3err.go
+49
View File
@@ -17,7 +17,11 @@ package s3err
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// APIError structure
@@ -140,6 +144,9 @@ const (
ErrInvalidVersionId
ErrNoSuchVersion
ErrSuspendedVersioningNotAllowed
ErrMultipleChecksumHeaders
ErrInvalidChecksumAlgorithm
ErrInvalidChecksumPart
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -584,6 +591,21 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMultipleChecksumHeaders: {
Code: "InvalidRequest",
Description: "Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumAlgorithm: {
Code: "InvalidRequest",
Description: "Checksum algorithm provided is unsupported. Please try again with any of the valid types: [CRC32, CRC32C, SHA1, SHA256]",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidChecksumPart: {
Code: "InvalidArgument",
Description: "Invalid Base64 or multiple checksums present in request",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
@@ -678,3 +700,30 @@ func encodeResponse(response interface{}) []byte {
e.Encode(response)
return bytesBuffer.Bytes()
}
// Returns invalid checksum error with the provided header in the error description
func GetInvalidChecksumHeaderErr(header string) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("Value for %v header is invalid.", header),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Returns checksum type mismatch APIError
func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIError {
return APIError{
Code: "InvalidRequest",
Description: fmt.Sprintf("Checksum Type mismatch occurred, expected checksum Type: %v, actual checksum Type: %v", expected, actual),
HTTPStatusCode: http.StatusBadRequest,
}
}
// Return incorrect checksum APIError
func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError {
return APIError{
Code: "BadDigest",
Description: fmt.Sprintf("The %v you specified did not match the calculated checksum.", strings.ToLower(string(algo))),
HTTPStatusCode: http.StatusBadRequest,
}
}
s3response/s3response.go
+56 -22
View File
@@ -29,16 +29,24 @@ const (
)
type PutObjectOutput struct {
ETag string
VersionID string
ETag string
VersionID string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
}
// Part describes part metadata.
type Part struct {
PartNumber int
LastModified time.Time
ETag string
Size int64
PartNumber int
LastModified time.Time
ETag string
Size int64
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
}
func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -59,9 +67,10 @@ func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
ChecksumAlgorithm types.ChecksumAlgorithm
Initiator Initiator
Owner Owner
@@ -101,6 +110,7 @@ type GetObjectAttributesResponse struct {
ObjectSize *int64
StorageClass types.StorageClass `xml:",omitempty"`
ObjectParts *ObjectParts
Checksum *types.Checksum
// Not included in the response body
VersionId *string
@@ -169,13 +179,14 @@ type ListObjectsV2Result struct {
}
type Object struct {
ETag *string
Key *string
LastModified *time.Time
Owner *types.Owner
RestoreStatus *types.RestoreStatus
Size *int64
StorageClass types.ObjectStorageClass
ChecksumAlgorithm []types.ChecksumAlgorithm
ETag *string
Key *string
LastModified *time.Time
Owner *types.Owner
RestoreStatus *types.RestoreStatus
Size *int64
StorageClass types.ObjectStorageClass
}
func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -197,12 +208,13 @@ func (o Object) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// Upload describes in progress multipart upload
type Upload struct {
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass types.StorageClass
Initiated time.Time
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass types.StorageClass
Initiated time.Time
ChecksumAlgorithm types.ChecksumAlgorithm
}
func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -332,6 +344,19 @@ type CopyObjectResult struct {
CopySourceVersionId string `xml:"-"`
}
type CopyPartResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
LastModified time.Time
ETag *string
ChecksumCRC32 *string
ChecksumCRC32C *string
ChecksumSHA1 *string
ChecksumSHA256 *string
// not included in the body
CopySourceVersionId string `xml:"-"`
}
func (r CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type Alias CopyObjectResult
aux := &struct {
@@ -467,3 +492,12 @@ func (AmzDate) ISO8601Parse(date string) (t time.Time, err error) {
type ListBucketsResult struct {
Buckets []Bucket
}
type Checksum struct {
Algorithms []types.ChecksumAlgorithm
CRC32 *string
CRC32C *string
SHA1 *string
SHA256 *string
}
tests/integration/group-tests.go
+33 -4
View File
@@ -138,6 +138,11 @@ func TestPutObject(s *S3Conf) {
PutObject_invalid_long_tags(s)
PutObject_missing_object_lock_retention_config(s)
PutObject_with_object_lock(s)
PutObject_checksum_algorithm_and_header_mismatch(s)
PutObject_multiple_checksum_headers(s)
PutObject_invalid_checksum_header(s)
PutObject_incorrect_checksums(s)
PutObject_checksums_success(s)
PutObject_success(s)
if !s.versioningEnabled {
PutObject_racey_success(s)
@@ -154,6 +159,8 @@ func TestHeadObject(s *S3Conf) {
HeadObject_non_existing_dir_object(s)
HeadObject_with_contenttype(s)
HeadObject_invalid_parent_dir(s)
HeadObject_not_enabled_checksum_mode(s)
HeadObject_checksums(s)
HeadObject_success(s)
}
@@ -164,6 +171,7 @@ func TestGetObjectAttributes(s *S3Conf) {
GetObjectAttributes_invalid_parent(s)
GetObjectAttributes_empty_attrs(s)
GetObjectAttributes_existing_object(s)
GetObjectAttributes_checksums(s)
}
func TestGetObject(s *S3Conf) {
@@ -173,6 +181,8 @@ func TestGetObject(s *S3Conf) {
GetObject_invalid_parent(s)
GetObject_with_meta(s)
GetObject_large_object(s)
GetObject_not_enabled_checksum_mode(s)
GetObject_checksums(s)
GetObject_success(s)
GetObject_directory_success(s)
GetObject_by_range_success(s)
@@ -191,6 +201,7 @@ func TestListObjects(s *S3Conf) {
ListObjects_max_keys_none(s)
ListObjects_marker_not_from_obj_list(s)
ListObjects_list_all_objs(s)
ListObjects_with_checksum(s)
}
func TestListObjectsV2(s *S3Conf) {
@@ -203,6 +214,7 @@ func TestListObjectsV2(s *S3Conf) {
ListObjectsV2_truncated_common_prefixes(s)
ListObjectsV2_all_objs_max_keys(s)
ListObjectsV2_list_all_objs(s)
ListObjectsV2_with_checksum(s)
ListObjectsV2_invalid_parent_prefix(s)
}
@@ -233,6 +245,11 @@ func TestCopyObject(s *S3Conf) {
CopyObject_to_itself_with_new_metadata(s)
CopyObject_CopySource_starting_with_slash(s)
CopyObject_non_existing_dir_object(s)
CopyObject_invalid_checksum_algorithm(s)
CopyObject_create_checksum_on_copy(s)
CopyObject_should_copy_the_existing_checksum(s)
CopyObject_should_replace_the_existing_checksum(s)
CopyObject_to_itself_by_replacing_the_checksum(s)
CopyObject_success(s)
}
@@ -265,6 +282,8 @@ func TestCreateMultipartUpload(s *S3Conf) {
CreateMultipartUpload_with_object_lock_not_enabled(s)
CreateMultipartUpload_with_object_lock_invalid_retention(s)
CreateMultipartUpload_past_retain_until_date(s)
CreateMultipartUpload_invalid_checksum_algorithm(s)
CreateMultipartUpload_valid_checksum_algorithm(s)
CreateMultipartUpload_success(s)
}
@@ -273,6 +292,15 @@ func TestUploadPart(s *S3Conf) {
UploadPart_invalid_part_number(s)
UploadPart_non_existing_key(s)
UploadPart_non_existing_mp_upload(s)
UploadPart_checksum_algorithm_and_header_mismatch(s)
UploadPart_multiple_checksum_headers(s)
UploadPart_invalid_checksum_header(s)
UploadPart_checksum_algorithm_mistmatch_on_initialization(s)
UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s)
UploadPart_required_checksum(s)
UploadPart_null_checksum(s)
UploadPart_incorrect_checksums(s)
UploadPart_with_checksums_success(s)
UploadPart_success(s)
}
@@ -288,6 +316,9 @@ func TestUploadPartCopy(s *S3Conf) {
UploadPartCopy_by_range_invalid_range(s)
UploadPartCopy_greater_range_than_obj_size(s)
UploadPartCopy_by_range_success(s)
UploadPartCopy_should_copy_the_checksum(s)
UploadPartCopy_should_not_copy_the_checksum(s)
UploadPartCopy_should_calculate_the_checksum(s)
}
func TestListParts(s *S3Conf) {
@@ -296,6 +327,7 @@ func TestListParts(s *S3Conf) {
ListParts_invalid_max_parts(s)
ListParts_default_max_parts(s)
ListParts_truncated(s)
ListParts_with_checksums(s)
ListParts_success(s)
}
@@ -306,6 +338,7 @@ func TestListMultipartUploads(s *S3Conf) {
ListMultipartUploads_max_uploads(s)
ListMultipartUploads_incorrect_next_key_marker(s)
ListMultipartUploads_ignore_upload_id_marker(s)
ListMultipartUploads_with_checksums(s)
ListMultipartUploads_success(s)
}
@@ -706,7 +739,6 @@ func GetIntTests() IntTests {
"ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets,
"ListBuckets_truncated": ListBuckets_truncated,
"ListBuckets_success": ListBuckets_success,
"ListBuckets_empty_success": ListBuckets_empty_success,
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
@@ -780,11 +812,9 @@ func GetIntTests() IntTests {
"ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes,
"ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys,
"ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs,
"ListObjectsV2_invalid_parent_prefix": ListObjectsV2_invalid_parent_prefix,
"ListObjectVersions_VD_success": ListObjectVersions_VD_success,
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
"DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash,
"DeleteObject_directory_not_empty": DeleteObject_directory_not_empty,
"DeleteObject_name_too_long": DeleteObject_name_too_long,
"DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object,
"DeleteObject_success": DeleteObject_success,
@@ -969,7 +999,6 @@ func GetIntTests() IntTests {
"IAM_userplus_CreateBucket": IAM_userplus_CreateBucket,
"IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner,
"IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root,
"IAM_ListBuckets": IAM_ListBuckets,
"AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied,
"AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied,
"AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access,
tests/integration/tests.go
+1462
View File
File diff suppressed because it is too large Load Diff
tests/integration/utils.go
+86 -15
View File
@@ -18,11 +18,15 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
"math/big"
rnd "math/rand"
@@ -475,11 +479,26 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client)
}, nil
}
func createMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
type mpCfg struct {
checksumAlgorithm types.ChecksumAlgorithm
}
type mpOpt func(*mpCfg)
func withChecksum(algo types.ChecksumAlgorithm) mpOpt {
return func(mc *mpCfg) { mc.checksumAlgorithm = algo }
}
func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) {
cfg := new(mpCfg)
for _, opt := range opts {
opt(cfg)
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &key,
Bucket: &bucket,
Key: &key,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
cancel()
return out, err
@@ -513,6 +532,9 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
if item.StorageClass != list2[i].StorageClass {
return false
}
if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm {
return false
}
}
return true
@@ -530,6 +552,21 @@ func compareParts(parts1, parts2 []types.Part) bool {
if *prt.ETag != *parts2[i].ETag {
return false
}
if *prt.Size != *parts2[i].Size {
return false
}
if getString(prt.ChecksumCRC32) != getString(parts2[i].ChecksumCRC32) {
return false
}
if getString(prt.ChecksumCRC32C) != getString(parts2[i].ChecksumCRC32C) {
return false
}
if getString(prt.ChecksumSHA1) != getString(parts2[i].ChecksumSHA1) {
return false
}
if getString(prt.ChecksumSHA256) != getString(parts2[i].ChecksumSHA256) {
return false
}
}
return true
}
@@ -647,6 +684,13 @@ func compareObjects(list1, list2 []types.Object) bool {
*obj.Key, *list2[i].Key, obj.StorageClass, list2[i].StorageClass)
return false
}
if len(obj.ChecksumAlgorithm) != 0 {
if obj.ChecksumAlgorithm[0] != list2[i].ChecksumAlgorithm[0] {
fmt.Printf("checksum algorithms are not equal: (%q %q) %v != %v\n",
*obj.Key, *list2[i].Key, obj.ChecksumAlgorithm[0], list2[i].ChecksumAlgorithm[0])
return false
}
}
}
return true
@@ -711,10 +755,28 @@ func compareDelObjects(list1, list2 []types.DeletedObject) bool {
return true
}
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string) (parts []types.Part, csum string, err error) {
func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId string, opts ...mpOpt) (parts []types.Part, csum string, err error) {
partSize := size / partCount
hash := sha256.New()
var hash hash.Hash
cfg := new(mpCfg)
for _, opt := range opts {
opt(cfg)
}
switch cfg.checksumAlgorithm {
case types.ChecksumAlgorithmCrc32:
hash = crc32.NewIEEE()
case types.ChecksumAlgorithmCrc32c:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case types.ChecksumAlgorithmSha1:
hash = sha1.New()
case types.ChecksumAlgorithmSha256:
hash = sha256.New()
default:
hash = sha256.New()
}
for partNumber := int64(1); partNumber <= partCount; partNumber++ {
partStart := (partNumber - 1) * partSize
@@ -730,11 +792,12 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
ChecksumAlgorithm: cfg.checksumAlgorithm,
})
cancel()
if err != nil {
@@ -742,14 +805,22 @@ func uploadParts(client *s3.Client, size, partCount int64, bucket, key, uploadId
}
parts = append(parts, types.Part{
ETag: out.ETag,
PartNumber: &pn,
Size: &partSize,
ETag: out.ETag,
PartNumber: &pn,
Size: &partSize,
ChecksumCRC32: out.ChecksumCRC32,
ChecksumCRC32C: out.ChecksumCRC32C,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
})
}
sum := hash.Sum(nil)
csum = hex.EncodeToString(sum[:])
if cfg.checksumAlgorithm == "" {
csum = hex.EncodeToString(sum[:])
} else {
csum = base64.StdEncoding.EncodeToString(sum[:])
}
return parts, csum, err
}