Merge pull request #1522 from versity/sis/conditional-writes

feat: implement conditional writes
This commit is contained in:
Ben McClelland
2025-09-10 08:54:04 -07:00
committed by GitHub
12 changed files with 1111 additions and 160 deletions
+116 -6
View File
@@ -301,6 +301,11 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s
return s3response.PutObjectOutput{}, err
}
err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch)
if err != nil {
return s3response.PutObjectOutput{}, err
}
metadata := parseMetadata(po.Metadata)
// Store the "Expires" property in the object metadata
@@ -850,6 +855,42 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
}
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil {
// evaluate the preconditions before deleting the object
props, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: input.Bucket,
Key: input.Key,
})
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
// if object doesn't exist, skip preconditions
// if unexpected error shows up, return the error
return nil, err
}
if err == nil {
var etag string
if props.ETag != nil {
etag = *props.ETag
}
var lastMod time.Time
if props.LastModified != nil {
lastMod = *props.LastModified
}
var size int64
if props.ContentLength != nil {
size = *props.ContentLength
}
err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size,
backend.ObjectDeletePreconditions{
IfMatch: input.IfMatch,
IfMatchLastModTime: input.IfMatchLastModifiedTime,
IfMatchSize: input.IfMatchSize,
})
if err != nil {
return nil, err
}
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
azerr, ok := err.(*azcore.ResponseError)
@@ -899,6 +940,26 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) {
_, err = az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &srcBucket,
Key: &srcObj,
IfMatch: input.CopySourceIfMatch,
IfNoneMatch: input.CopySourceIfNoneMatch,
IfModifiedSince: input.CopySourceIfModifiedSince,
IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
}
if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
if input.MetadataDirective != types.MetadataDirectiveReplace {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
@@ -977,11 +1038,6 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
}, nil
}
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
// Get the source object
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
if err != nil {
@@ -1210,7 +1266,7 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyPartResult{}, nil
return s3response.CopyPartResult{}, err
}
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
@@ -1413,6 +1469,22 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
// Cleans up the initiated multipart upload in .sgwtmp namespace
func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
if input.IfMatchInitiatedTime != nil {
client, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
return err
}
resp, err := client.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil)
if err != nil {
return parseMpError(err)
@@ -1440,6 +1512,11 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip
func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
var res s3response.CompleteMultipartUploadResult
err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch)
if err != nil {
return s3response.CompleteMultipartUploadResult{}, "", err
}
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
@@ -2058,6 +2135,29 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
return nil
}
func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error {
if areNils(ifMatch, ifNoneMatch) {
return nil
}
// call HeadObject to evaluate preconditions
// if object doesn't exist, move forward with the object creation
// otherwise return the error
_, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: bucket,
Key: object,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
if errors.Is(err, s3err.GetAPIError(s3err.ErrNotModified)) {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return err
}
return nil
}
func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
data, ok := meta[string(key)]
if !ok {
@@ -2105,3 +2205,13 @@ func convertAzureEtag(etag *azcore.ETag) string {
return *backend.TrimEtag(str) + "-1"
}
func areNils[T any](args ...*T) bool {
for _, arg := range args {
if arg != nil {
return false
}
}
return true
}
+38
View File
@@ -532,3 +532,41 @@ func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreCond
return nil
}
// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions
func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error {
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
if ifNoneMatch != nil && *ifNoneMatch == etag {
return errPreconditionFailed
}
return nil
}
type ObjectDeletePreconditions struct {
IfMatch *string
IfMatchLastModTime *time.Time
IfMatchSize *int64
}
// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject
func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error {
ifMatch := preconditions.IfMatch
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
ifMatchTime := preconditions.IfMatchLastModTime
if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() {
return errPreconditionFailed
}
ifMatchSize := preconditions.IfMatchSize
if ifMatchSize != nil && *ifMatchSize != size {
return errPreconditionFailed
}
return nil
}
+110 -10
View File
@@ -1401,6 +1401,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
return res, "", err
}
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
if err == nil {
err = backend.EvaluateMatchPreconditions(string(b), input.IfMatch, input.IfNoneMatch)
if err != nil {
return res, "", err
}
}
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
@@ -2009,11 +2017,17 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err = os.Stat(filepath.Join(objdir, uploadID))
f, err := os.Stat(filepath.Join(objdir, uploadID))
if err != nil {
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if mpu.IfMatchInitiatedTime != nil {
if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
}
err = os.RemoveAll(filepath.Join(objdir, uploadID))
if err != nil {
return fmt.Errorf("remove multipart upload container: %w", err)
@@ -2600,6 +2614,32 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, err
}
srcf, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
}
defer srcf.Close()
// evaluate preconditions
b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey)
srcEtag := string(b)
if err != nil {
srcEtag = ""
}
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
IfMatch: upi.CopySourceIfMatch,
IfNoneMatch: upi.CopySourceIfNoneMatch,
IfModSince: upi.CopySourceIfModifiedSince,
IfUnmodeSince: upi.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyPartResult{}, err
}
f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
*upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile)
if err != nil {
@@ -2610,15 +2650,6 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
defer f.cleanup()
srcf, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
}
defer srcf.Close()
rdr := io.NewSectionReader(srcf, startOffset, length)
hash := md5.New()
tr := io.TeeReader(rdr, hash)
@@ -2748,6 +2779,15 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3
name := filepath.Join(*po.Bucket, *po.Key)
// evaluate preconditions
etagBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, etagkey)
if err == nil {
err := backend.EvaluateMatchPreconditions(string(etagBytes), po.IfMatch, po.IfNoneMatch)
if err != nil {
return s3response.PutObjectOutput{}, err
}
}
uid, gid, doChown := p.getChownIDs(acct)
contentLength := int64(0)
@@ -3073,6 +3113,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, err
}
evalPreconditions := func(f os.FileInfo, bucket, object string) error {
var err error
if f == nil {
f, err = os.Stat(filepath.Join(bucket, object))
if err != nil {
return nil
}
}
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
etag := string(b)
if err != nil {
etag = ""
}
// evaluate preconditions
return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(),
backend.ObjectDeletePreconditions{
IfMatch: input.IfMatch,
IfMatchLastModTime: input.IfMatchLastModifiedTime,
IfMatchSize: input.IfMatchSize,
})
}
// Directory objects can't have versions
if !isDir && p.versioningEnabled() && vStatus != "" {
if getString(input.VersionId) == "" {
@@ -3089,6 +3153,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
err = evalPreconditions(fi, bucket, object)
if err != nil {
return nil, err
}
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
acct = auth.Account{}
@@ -3148,6 +3217,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}
if string(vId) == *input.VersionId {
// evaluate preconditions
err := evalPreconditions(nil, bucket, object)
if err != nil {
return nil, err
}
// if the specified VersionId is the same as in the latest version,
// remove the latest version, find the latest version from the versioning
// directory and move to the place of the deleted object, to make it the latest
@@ -3242,6 +3316,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}, nil
}
err = evalPreconditions(nil, versionPath, *input.VersionId)
if err != nil {
return nil, err
}
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
@@ -3289,6 +3368,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return &s3.DeleteObjectOutput{}, nil
}
err = evalPreconditions(fi, bucket, object)
if err != nil {
return nil, err
}
err = os.Remove(objpath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
@@ -4031,6 +4115,22 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey)
srcEtag := string(b)
if err != nil {
srcEtag = ""
}
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
IfMatch: input.CopySourceIfMatch,
IfNoneMatch: input.CopySourceIfNoneMatch,
IfModSince: input.CopySourceIfModifiedSince,
IfUnmodeSince: input.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
mdmap := make(map[string]string)
p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap)
+14 -6
View File
@@ -69,6 +69,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
uploadId := ctx.Query("uploadId")
ifMatchInitiatedTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Initiated-Time"))
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
@@ -96,9 +97,10 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
err = c.be.AbortMultipartUpload(ctx.Context(),
&s3.AbortMultipartUploadInput{
UploadId: &uploadId,
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Bucket: &bucket,
Key: &key,
IfMatchInitiatedTime: ifMatchInitiatedTime,
})
return &Response{
MetaOpts: &MetaOptions{
@@ -113,6 +115,9 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")
ifMatch := utils.GetStringPtr(ctx.Get("If-Match"))
ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time"))
ifMatchSize := utils.ParseIfMatchSize(ctx)
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
@@ -165,9 +170,12 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
res, err := c.be.DeleteObject(ctx.Context(),
&s3.DeleteObjectInput{
Bucket: &bucket,
Key: &key,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
VersionId: &versionId,
IfMatch: ifMatch,
IfMatchLastModifiedTime: ifMatchLastModTime,
IfMatchSize: ifMatchSize,
})
if err != nil {
return &Response{
+4
View File
@@ -323,6 +323,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
}, err
}
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -338,6 +340,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumType: checksumType,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
return &Response{
Data: res,
+22 -44
View File
@@ -21,7 +21,6 @@ import (
"io"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -355,14 +354,20 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) {
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
resp, err := c.be.UploadPartCopy(ctx.Context(),
&s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &key,
CopySource: &copySource,
PartNumber: &partNumber,
UploadId: &uploadId,
CopySourceRange: &copySrcRange,
Bucket: &bucket,
Key: &key,
CopySource: &copySource,
PartNumber: &partNumber,
UploadId: &uploadId,
CopySourceRange: &copySrcRange,
CopySourceIfMatch: preconditionHdrs.IfMatch,
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
})
var headers map[string]*string
if err == nil && resp.CopySourceVersionId != "" {
@@ -434,10 +439,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/")
copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match")
copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match")
copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since")
copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since")
metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy)))
taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy)))
contentType := ctx.Get("Content-Type")
@@ -480,35 +481,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
}, err
}
var mtime *time.Time
if copySrcModifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcModifSince)
if err != nil {
debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err)
// TODO: check the error type for invalid values
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
mtime = &tm
}
var umtime *time.Time
if copySrcUnmodifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
if err != nil {
debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err)
// TODO: check the error type for invalid values
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
umtime = &tm
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace {
@@ -548,6 +520,8 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
}, err
}
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
res, err := c.be.CopyObject(ctx.Context(),
s3response.CopyObjectInput{
Bucket: &bucket,
@@ -561,10 +535,10 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
Tagging: &tagging,
TaggingDirective: taggingDirective,
CopySource: &copySource,
CopySourceIfMatch: &copySrcIfMatch,
CopySourceIfNoneMatch: &copySrcIfNoneMatch,
CopySourceIfModifiedSince: mtime,
CopySourceIfUnmodifiedSince: umtime,
CopySourceIfMatch: preconditionHdrs.IfMatch,
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
ExpectedBucketOwner: &acct.Access,
Metadata: metadata,
MetadataDirective: metaDirective,
@@ -691,6 +665,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
body = bytes.NewReader([]byte{})
}
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
res, err := c.be.PutObject(ctx.Context(),
s3response.PutObjectInput{
Bucket: &bucket,
@@ -714,6 +690,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
return &Response{
Headers: map[string]*string{
-36
View File
@@ -836,42 +836,6 @@ func TestS3ApiController_CopyObject(t *testing.T) {
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid copy source modified since",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Copy-Source": "bucket/object",
"X-Amz-Copy-Source-If-Modified-Since": "invalid_date",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid copy source unmodified since",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Copy-Source": "bucket/object",
"X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid metadata directive",
input: testInput{
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"strconv"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/debuglogger"
)
// ConditionalHeaders holds the conditional header values
type ConditionalHeaders struct {
IfMatch *string
IfNoneMatch *string
IfModSince *time.Time
IfUnmodeSince *time.Time
}
type precondtionCfg struct {
withCopySource bool
}
type preconditionOpt func(*precondtionCfg)
func WithCopySource() preconditionOpt {
return func(o *precondtionCfg) { o.withCopySource = true }
}
// ParsePreconditionHeaders parses the precondition headers:
// - If-Match
// - If-None-Match
// - If-Modified-Since
// - If-Unmodified-Since
func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders {
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...)
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...)
return ConditionalHeaders{
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
IfModSince: ifModSince,
IfUnmodeSince: ifUnmodeSince,
}
}
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
}
prefix := ""
if cfg.withCopySource {
prefix = "X-Amz-Copy-Source-"
}
return GetStringPtr(ctx.Get(prefix + "If-Match")), GetStringPtr(ctx.Get(prefix + "If-None-Match"))
}
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
// headers from fiber context to *time.Time
func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
}
prefix := ""
if cfg.withCopySource {
prefix = "X-Amz-Copy-Source-"
}
ifModSince := ctx.Get(prefix + "If-Modified-Since")
ifUnmodSince := ctx.Get(prefix + "If-Unmodified-Since")
ifModSinceParsed := ParsePreconditionDateHeader(ifModSince)
ifUnmodSinceParsed := ParsePreconditionDateHeader(ifUnmodSince)
return ifModSinceParsed, ifUnmodSinceParsed
}
// ParsePreconditionDateHeader tries to parse the given date string as
// - RFC1123
// - RFC3339
// both are valid
func ParsePreconditionDateHeader(date string) *time.Time {
if date == "" {
return nil
}
// try to parse as RFC1123
parsed, err := time.Parse(time.RFC1123, date)
if err == nil {
// ignore future dates
if parsed.After(time.Now()) {
return nil
}
return &parsed
}
// try to parse as RFC3339
parsed, err = time.Parse(time.RFC3339, date)
if err == nil {
// ignore future dates
if parsed.After(time.Now()) {
return nil
}
return &parsed
}
return nil
}
// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64
// if parsing fails, returns nil
func ParseIfMatchSize(ctx *fiber.Ctx) *int64 {
ifMatchSizeHdr := ctx.Get("x-amz-if-match-size")
ifMatchSize, err := strconv.ParseInt(ifMatchSizeHdr, 10, 64)
if err != nil {
debuglogger.Logf("failed to parse 'x-amz-if-match-size': %s", ifMatchSizeHdr)
return nil
}
return &ifMatchSize
}
-53
View File
@@ -642,59 +642,6 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type
return algo, chType, nil
}
// ConditionalHeaders holds the conditional header values
type ConditionalHeaders struct {
IfMatch *string
IfNoneMatch *string
IfModSince *time.Time
IfUnmodeSince *time.Time
}
// ParsePreconditionHeaders parses the precondition headers:
// - If-Match
// - If-None-Match
// - If-Modified-Since
// - If-Unmodified-Since
func ParsePreconditionHeaders(ctx *fiber.Ctx) ConditionalHeaders {
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx)
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx)
return ConditionalHeaders{
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
IfModSince: ifModSince,
IfUnmodeSince: ifUnmodeSince,
}
}
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx) (*string, *string) {
return GetStringPtr(ctx.Get("If-Match")), GetStringPtr(ctx.Get("If-None-Match"))
}
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
// headers from fiber context to *time.Time
func ParsePreconditionDateHeaders(ctx *fiber.Ctx) (*time.Time, *time.Time) {
ifModSince := ctx.Get("If-Modified-Since")
ifUnmodSince := ctx.Get("If-Unmodified-Since")
var ifModSinceParsed, ifUnmodSinceParsed *time.Time
// the time format should be a valid RFC1123
// if parsing fails, ignore the error and leave the value as nil
modParsed, err := time.Parse(time.RFC1123, ifModSince)
if err == nil {
ifModSinceParsed = &modParsed
}
unmodParsed, err := time.Parse(time.RFC1123, ifUnmodSince)
if err == nil {
ifUnmodSinceParsed = &unmodParsed
}
return ifModSinceParsed, ifUnmodSinceParsed
}
// TagLimit specifies the allowed tag count in a tag set
type TagLimit int
+12
View File
@@ -149,6 +149,7 @@ func TestPutObject(s *S3Conf) {
PutObject_with_object_lock(s)
PutObject_invalid_legal_hold(s)
PutObject_invalid_object_lock_mode(s)
PutObject_conditional_writes(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
PutObject_checksum_algorithm_and_header_mismatch(s)
@@ -279,6 +280,7 @@ func TestDeleteObject(s *S3Conf) {
DeleteObject_non_existing_dir_object(s)
DeleteObject_directory_object(s)
DeleteObject_non_empty_dir_obj(s)
DeleteObject_conditional_writes(s)
DeleteObject_success(s)
DeleteObject_success_status_code(s)
}
@@ -307,6 +309,7 @@ func TestCopyObject(s *S3Conf) {
CopyObject_invalid_object_lock_mode(s)
CopyObject_with_legal_hold(s)
CopyObject_with_retention_lock(s)
CopyObject_conditional_reads(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
CopyObject_invalid_checksum_algorithm(s)
@@ -398,6 +401,7 @@ func TestUploadPartCopy(s *S3Conf) {
UploadPartCopy_should_copy_the_checksum(s)
UploadPartCopy_should_not_copy_the_checksum(s)
UploadPartCopy_should_calculate_the_checksum(s)
UploadPartCopy_conditional_reads(s)
}
}
@@ -437,6 +441,7 @@ func TestAbortMultipartUpload(s *S3Conf) {
AbortMultipartUpload_incorrect_object_key(s)
AbortMultipartUpload_success(s)
AbortMultipartUpload_success_status_code(s)
AbortMultipartUpload_if_match_initiated_time(s)
}
func TestCompleteMultipartUpload(s *S3Conf) {
@@ -448,6 +453,7 @@ func TestCompleteMultipartUpload(s *S3Conf) {
CompleteMultipartUpload_empty_parts(s)
CompleteMultipartUpload_incorrect_parts_order(s)
CompleteMultipartUpload_mpu_object_size(s)
CompleteMultipartUpload_conditional_writes(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
CompleteMultipartUpload_invalid_checksum_type(s)
@@ -1053,6 +1059,7 @@ func GetIntTests() IntTests {
"PutObject_with_object_lock": PutObject_with_object_lock,
"PutObject_invalid_legal_hold": PutObject_invalid_legal_hold,
"PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode,
"PutObject_conditional_writes": PutObject_conditional_writes,
"PutObject_invalid_credentials": PutObject_invalid_credentials,
"PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch,
"PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers,
@@ -1184,6 +1191,7 @@ func GetIntTests() IntTests {
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
"DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash,
"DeleteObject_non_empty_dir_obj": DeleteObject_non_empty_dir_obj,
"DeleteObject_conditional_writes": DeleteObject_conditional_writes,
"DeleteObject_name_too_long": DeleteObject_name_too_long,
"CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object,
"CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object,
@@ -1211,6 +1219,7 @@ func GetIntTests() IntTests {
"CopyObject_invalid_object_lock_mode": CopyObject_invalid_object_lock_mode,
"CopyObject_with_legal_hold": CopyObject_with_legal_hold,
"CopyObject_with_retention_lock": CopyObject_with_retention_lock,
"CopyObject_conditional_reads": CopyObject_conditional_reads,
"CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm,
"CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy,
"CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum,
@@ -1270,6 +1279,7 @@ func GetIntTests() IntTests {
"UploadPartCopy_exceeding_copy_source_range": UploadPartCopy_exceeding_copy_source_range,
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
"UploadPartCopy_conditional_reads": UploadPartCopy_conditional_reads,
"UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum,
"UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum,
"UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum,
@@ -1295,6 +1305,7 @@ func GetIntTests() IntTests {
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
"AbortMultipartUpload_success": AbortMultipartUpload_success,
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
"AbortMultipartUpload_if_match_initiated_time": AbortMultipartUpload_if_match_initiated_time,
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
@@ -1302,6 +1313,7 @@ func GetIntTests() IntTests {
"CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts,
"CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order,
"CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size,
"CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes,
"CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type,
"CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part,
"CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part,
+648
View File
@@ -3043,6 +3043,7 @@ func PutObject_invalid_legal_hold(s *S3Conf) error {
return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus))
}, withLock())
}
func PutObject_invalid_object_lock_mode(s *S3Conf) error {
testName := "PutObject_invalid_object_lock_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -3057,6 +3058,78 @@ func PutObject_invalid_object_lock_mode(s *S3Conf) error {
}, withLock())
}
func PutObject_conditional_writes(s *S3Conf) error {
testName := "PutObject_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
etag := res.res.ETag
incorrectEtag := getPtr("incorrect_etag")
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
obj string
ifMatch *string
ifNoneMatch *string
err error
}{
{obj, etag, nil, nil},
{obj, etag, etag, errPrecond},
{obj, etag, incorrectEtag, nil},
{obj, incorrectEtag, incorrectEtag, errPrecond},
{obj, incorrectEtag, etag, errPrecond},
{obj, incorrectEtag, nil, errPrecond},
{obj, nil, incorrectEtag, nil},
{obj, nil, etag, errPrecond},
{obj, nil, nil, nil},
// should ignore the precondition headers if
// an object with the given name doesn't exist
{"obj-1", incorrectEtag, etag, nil},
{"obj-2", etag, etag, nil},
{"obj-3", etag, incorrectEtag, nil},
{"obj-4", incorrectEtag, nil, nil},
{"obj-5", nil, etag, nil},
} {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &test.obj,
Body: bytes.NewReader([]byte("dummy")),
IfMatch: test.ifMatch,
IfNoneMatch: test.ifNoneMatch,
}, s3client)
if err == nil {
// azure blob storage generates different ETags for
// the exact same data.
// to avoid ETag collision reassign the etag value
*etag = *res.res.ETag
}
if test.err == nil && err != nil {
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
}
if test.err != nil {
apierr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
}
if err := checkApiErr(err, apierr); err != nil {
return fmt.Errorf("test case %v: %w", i, err)
}
}
}
return nil
})
}
func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error {
testName := "PutObject_checksum_algorithm_and_header_mismatch"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -6895,6 +6968,98 @@ func DeleteObject_non_empty_dir_obj(s *S3Conf) error {
})
}
func DeleteObject_conditional_writes(s *S3Conf) error {
testName := "DeleteObject_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
var etag *string = getPtr("")
var size *int64 = getPtr(int64(0))
var modTime *time.Time = getPtr(time.Now())
createObj := func() error {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
// get the exact LastModified time
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
*etag = *res.res.ETag
*size = *res.res.Size
*modTime = *out.LastModified
return nil
}
err := createObj()
if err != nil {
return err
}
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
ifMatch *string
size *int64
modTime *time.Time
err error
}{
// no error cases
{etag, size, modTime, nil},
{etag, nil, nil, nil},
{nil, size, nil, nil},
{nil, nil, modTime, nil},
{etag, size, nil, nil},
{etag, nil, modTime, nil},
{nil, size, modTime, nil},
// error cases
{getPtr("incorrect_etag"), nil, nil, errPrecond},
{nil, getPtr(int64(23234)), nil, errPrecond},
{nil, nil, getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
{getPtr("incorrect_etag"), getPtr(int64(23234)), nil, errPrecond},
{getPtr("incorrect_etag"), getPtr(int64(23234)), getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
} {
err := createObj()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: &bucket,
Key: &obj,
IfMatch: test.ifMatch,
IfMatchSize: test.size,
IfMatchLastModifiedTime: test.modTime,
})
cancel()
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func DeleteObject_directory_not_empty(s *S3Conf) error {
testName := "DeleteObject_directory_not_empty"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -8022,6 +8187,156 @@ func CopyObject_with_retention_lock(s *S3Conf) error {
}, withLock())
}
func CopyObject_conditional_reads(s *S3Conf) error {
testName := "CopyObject_conditional_reads"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
key := "my-obj"
obj, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
}, s3client)
if err != nil {
return err
}
errMod := s3err.GetAPIError(s3err.ErrNotModified)
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
// sleep one second to get dates before and after
// the object creation
time.Sleep(time.Second * 1)
before := time.Now().AddDate(0, 0, -3)
after := time.Now()
etag := obj.res.ETag
for i, test := range []struct {
ifmatch *string
ifnonematch *string
ifmodifiedsince *time.Time
ifunmodifiedsince *time.Time
err error
}{
// all the cases when preconditions are either empty, true or false
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
{getPtr("invalid_etag"), etag, &before, &before, errCond},
{getPtr("invalid_etag"), etag, &before, &after, errCond},
{getPtr("invalid_etag"), etag, &before, nil, errCond},
{getPtr("invalid_etag"), etag, &after, &before, errCond},
{getPtr("invalid_etag"), etag, &after, &after, errCond},
{getPtr("invalid_etag"), etag, &after, nil, errCond},
{getPtr("invalid_etag"), etag, nil, &before, errCond},
{getPtr("invalid_etag"), etag, nil, &after, errCond},
{getPtr("invalid_etag"), etag, nil, nil, errCond},
{getPtr("invalid_etag"), nil, &before, &before, errCond},
{getPtr("invalid_etag"), nil, &before, &after, errCond},
{getPtr("invalid_etag"), nil, &before, nil, errCond},
{getPtr("invalid_etag"), nil, &after, &before, errCond},
{getPtr("invalid_etag"), nil, &after, &after, errCond},
{getPtr("invalid_etag"), nil, &after, nil, errCond},
{getPtr("invalid_etag"), nil, nil, &before, errCond},
{getPtr("invalid_etag"), nil, nil, &after, errCond},
{getPtr("invalid_etag"), nil, nil, nil, errCond},
{etag, getPtr("invalid_etag"), &before, &before, nil},
{etag, getPtr("invalid_etag"), &before, &after, nil},
{etag, getPtr("invalid_etag"), &before, nil, nil},
{etag, getPtr("invalid_etag"), &after, &before, nil},
{etag, getPtr("invalid_etag"), &after, &after, nil},
{etag, getPtr("invalid_etag"), &after, nil, nil},
{etag, getPtr("invalid_etag"), nil, &before, nil},
{etag, getPtr("invalid_etag"), nil, &after, nil},
{etag, getPtr("invalid_etag"), nil, nil, nil},
{etag, etag, &before, &before, errMod},
{etag, etag, &before, &after, errMod},
{etag, etag, &before, nil, errMod},
{etag, etag, &after, &before, errMod},
{etag, etag, &after, &after, errMod},
{etag, etag, &after, nil, errMod},
{etag, etag, nil, &before, errMod},
{etag, etag, nil, &after, errMod},
{etag, etag, nil, nil, errMod},
{etag, nil, &before, &before, nil},
{etag, nil, &before, &after, nil},
{etag, nil, &before, nil, nil},
{etag, nil, &after, &before, errMod},
{etag, nil, &after, &after, errMod},
{etag, nil, &after, nil, errMod},
{etag, nil, nil, &before, nil},
{etag, nil, nil, &after, nil},
{etag, nil, nil, nil, nil},
{nil, getPtr("invalid_etag"), &before, &before, errCond},
{nil, getPtr("invalid_etag"), &before, &after, nil},
{nil, getPtr("invalid_etag"), &before, nil, nil},
{nil, getPtr("invalid_etag"), &after, &before, errCond},
{nil, getPtr("invalid_etag"), &after, &after, nil},
{nil, getPtr("invalid_etag"), &after, nil, nil},
{nil, getPtr("invalid_etag"), nil, &before, errCond},
{nil, getPtr("invalid_etag"), nil, &after, nil},
{nil, getPtr("invalid_etag"), nil, nil, nil},
{nil, etag, &before, &before, errCond},
{nil, etag, &before, &after, errMod},
{nil, etag, &before, nil, errMod},
{nil, etag, &after, &before, errCond},
{nil, etag, &after, &after, errMod},
{nil, etag, &after, nil, errMod},
{nil, etag, nil, &before, errCond},
{nil, etag, nil, &after, errMod},
{nil, etag, nil, nil, errMod},
{nil, nil, &before, &before, errCond},
{nil, nil, &before, &after, nil},
{nil, nil, &before, nil, nil},
{nil, nil, &after, &before, errCond},
{nil, nil, &after, &after, errMod},
{nil, nil, &after, nil, errMod},
{nil, nil, nil, &before, errCond},
{nil, nil, nil, &after, nil},
{nil, nil, nil, nil, nil},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: getPtr("dst-obj"),
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
CopySourceIfMatch: test.ifmatch,
CopySourceIfNoneMatch: test.ifnonematch,
CopySourceIfModifiedSince: test.ifmodifiedsince,
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func CopyObject_invalid_checksum_algorithm(s *S3Conf) error {
testName := "CopyObject_invalid_checksum_algorithm"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -10476,6 +10791,164 @@ func UploadPartCopy_by_range_success(s *S3Conf) error {
})
}
func UploadPartCopy_conditional_reads(s *S3Conf) error {
testName := "UploadPartCopy_conditional_reads"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
key := "my-obj"
obj, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
}, s3client)
if err != nil {
return err
}
errMod := s3err.GetAPIError(s3err.ErrNotModified)
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
// sleep one second to get dates before and after
// the object creation
time.Sleep(time.Second * 1)
before := time.Now().AddDate(0, 0, -3)
after := time.Now()
etag := obj.res.ETag
for i, test := range []struct {
ifmatch *string
ifnonematch *string
ifmodifiedsince *time.Time
ifunmodifiedsince *time.Time
err error
}{
// all the cases when preconditions are either empty, true or false
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
{getPtr("invalid_etag"), etag, &before, &before, errCond},
{getPtr("invalid_etag"), etag, &before, &after, errCond},
{getPtr("invalid_etag"), etag, &before, nil, errCond},
{getPtr("invalid_etag"), etag, &after, &before, errCond},
{getPtr("invalid_etag"), etag, &after, &after, errCond},
{getPtr("invalid_etag"), etag, &after, nil, errCond},
{getPtr("invalid_etag"), etag, nil, &before, errCond},
{getPtr("invalid_etag"), etag, nil, &after, errCond},
{getPtr("invalid_etag"), etag, nil, nil, errCond},
{getPtr("invalid_etag"), nil, &before, &before, errCond},
{getPtr("invalid_etag"), nil, &before, &after, errCond},
{getPtr("invalid_etag"), nil, &before, nil, errCond},
{getPtr("invalid_etag"), nil, &after, &before, errCond},
{getPtr("invalid_etag"), nil, &after, &after, errCond},
{getPtr("invalid_etag"), nil, &after, nil, errCond},
{getPtr("invalid_etag"), nil, nil, &before, errCond},
{getPtr("invalid_etag"), nil, nil, &after, errCond},
{getPtr("invalid_etag"), nil, nil, nil, errCond},
{etag, getPtr("invalid_etag"), &before, &before, nil},
{etag, getPtr("invalid_etag"), &before, &after, nil},
{etag, getPtr("invalid_etag"), &before, nil, nil},
{etag, getPtr("invalid_etag"), &after, &before, nil},
{etag, getPtr("invalid_etag"), &after, &after, nil},
{etag, getPtr("invalid_etag"), &after, nil, nil},
{etag, getPtr("invalid_etag"), nil, &before, nil},
{etag, getPtr("invalid_etag"), nil, &after, nil},
{etag, getPtr("invalid_etag"), nil, nil, nil},
{etag, etag, &before, &before, errMod},
{etag, etag, &before, &after, errMod},
{etag, etag, &before, nil, errMod},
{etag, etag, &after, &before, errMod},
{etag, etag, &after, &after, errMod},
{etag, etag, &after, nil, errMod},
{etag, etag, nil, &before, errMod},
{etag, etag, nil, &after, errMod},
{etag, etag, nil, nil, errMod},
{etag, nil, &before, &before, nil},
{etag, nil, &before, &after, nil},
{etag, nil, &before, nil, nil},
{etag, nil, &after, &before, errMod},
{etag, nil, &after, &after, errMod},
{etag, nil, &after, nil, errMod},
{etag, nil, nil, &before, nil},
{etag, nil, nil, &after, nil},
{etag, nil, nil, nil, nil},
{nil, getPtr("invalid_etag"), &before, &before, errCond},
{nil, getPtr("invalid_etag"), &before, &after, nil},
{nil, getPtr("invalid_etag"), &before, nil, nil},
{nil, getPtr("invalid_etag"), &after, &before, errCond},
{nil, getPtr("invalid_etag"), &after, &after, nil},
{nil, getPtr("invalid_etag"), &after, nil, nil},
{nil, getPtr("invalid_etag"), nil, &before, errCond},
{nil, getPtr("invalid_etag"), nil, &after, nil},
{nil, getPtr("invalid_etag"), nil, nil, nil},
{nil, etag, &before, &before, errCond},
{nil, etag, &before, &after, errMod},
{nil, etag, &before, nil, errMod},
{nil, etag, &after, &before, errCond},
{nil, etag, &after, &after, errMod},
{nil, etag, &after, nil, errMod},
{nil, etag, nil, &before, errCond},
{nil, etag, nil, &after, errMod},
{nil, etag, nil, nil, errMod},
{nil, nil, &before, &before, errCond},
{nil, nil, &before, &after, nil},
{nil, nil, &before, nil, nil},
{nil, nil, &after, &before, errCond},
{nil, nil, &after, &after, errMod},
{nil, nil, &after, nil, errMod},
{nil, nil, nil, &before, errCond},
{nil, nil, nil, &after, nil},
{nil, nil, nil, nil, nil},
} {
mpKey := "mp-key"
mp, err := createMp(s3client, bucket, mpKey)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: mp.UploadId,
PartNumber: getPtr(int32(1)),
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
CopySourceIfMatch: test.ifmatch,
CopySourceIfNoneMatch: test.ifnonematch,
CopySourceIfModifiedSince: test.ifmodifiedsince,
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_copy_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -11427,6 +11900,87 @@ func AbortMultipartUpload_success_status_code(s *S3Conf) error {
})
}
func AbortMultipartUpload_if_match_initiated_time(s *S3Conf) error {
testName := "AbortMultipartUpload_if_match_initiated_time"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
var initiated *time.Time = getPtr(time.Now())
// createMpUpload creates a multipart uplod
// and retruns the uploadId and creation date
abortMp := func(date *time.Time) error {
mpObj := "my-obj"
mp, err := createMp(s3client, bucket, mpObj)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
var initiatedTime *time.Time
for _, up := range res.Uploads {
if getString(up.UploadId) == getString(mp.UploadId) {
initiatedTime = up.Initiated
break
}
}
if initiatedTime == nil {
return fmt.Errorf("unexpected err: the multipart upload is not found")
}
*initiated = *initiatedTime
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: &bucket,
Key: &mpObj,
UploadId: mp.UploadId,
IfMatchInitiatedTime: date,
})
cancel()
return err
}
for i, test := range []struct {
date *time.Time
err error
}{
{nil, nil},
// match: success case
{initiated, nil},
// should ignore future dates
{getPtr(initiated.AddDate(1, 0, 0)), nil},
// should fail if the initation date doesn't match
{getPtr(initiated.AddDate(-1, 0, 1)), s3err.GetAPIError(s3err.ErrPreconditionFailed)},
} {
err := abortMp(test.date)
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error {
testName := "CompletedMultipartUpload_non_existing_bucket"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -12593,6 +13147,100 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error {
})
}
func CompleteMultipartUpload_conditional_writes(s *S3Conf) error {
testName := "CompleteMultipartUpload_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
etag := getPtr("")
incorrectEtag := getPtr("incorrect_etag")
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
obj string
ifMatch *string
ifNoneMatch *string
err error
}{
{obj, etag, nil, nil},
{obj, etag, etag, errPrecond},
{obj, etag, incorrectEtag, nil},
{obj, incorrectEtag, incorrectEtag, errPrecond},
{obj, incorrectEtag, etag, errPrecond},
{obj, incorrectEtag, nil, errPrecond},
{obj, nil, incorrectEtag, nil},
{obj, nil, etag, errPrecond},
{obj, nil, nil, nil},
// should ignore the precondition headers if
// an object with the given name doesn't exist
{"obj-1", incorrectEtag, etag, nil},
{"obj-2", etag, etag, nil},
{"obj-3", etag, incorrectEtag, nil},
{"obj-4", incorrectEtag, nil, nil},
{"obj-5", nil, etag, nil},
} {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
// azure blob storage generates different ETags for
// the exact same data.
// to avoid ETag collision reassign the etag value
*etag = *res.res.ETag
mp, err := createMp(s3client, bucket, test.obj)
if err != nil {
return err
}
parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, test.obj, *mp.UploadId)
if err != nil {
return err
}
part := parts[0]
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &test.obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: []types.CompletedPart{
{
ETag: part.ETag,
PartNumber: getPtr(int32(1)),
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
},
},
},
IfMatch: test.ifMatch,
IfNoneMatch: test.ifNoneMatch,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
}
if test.err != nil {
apierr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
}
if err := checkApiErr(err, apierr); err != nil {
return fmt.Errorf("test case %v: %w", i, err)
}
}
}
return nil
})
}
func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error {
testName := "CompleteMultipartUpload_invalid_part_number"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+9 -5
View File
@@ -518,11 +518,15 @@ type putObjectOutput struct {
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
data := make([]byte, lgth)
rand.Read(data)
csum := sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
var csum [32]byte
var data []byte
if input.Body == nil {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := client.PutObject(ctx, input)