feat: implement conditional writes

Closes #821

**Implements conditional operations across object APIs:**

* **PutObject** and **CompleteMultipartUpload**:
  Supports conditional writes with `If-Match` and `If-None-Match` headers (ETag comparisons).
  Evaluation is based on an existing object with the same key in the bucket. The operation is allowed only if the preconditions are satisfied. If no object exists for the key, these headers are ignored.

* **CopyObject** and **UploadPartCopy**:
  Adds conditional reads on the copy source object with the following headers:

  * `x-amz-copy-source-if-match`
  * `x-amz-copy-source-if-none-match`
  * `x-amz-copy-source-if-modified-since`
  * `x-amz-copy-source-if-unmodified-since`
    The first two are ETag comparisons, while the latter two compare against the copy source’s `LastModified` timestamp.

* **AbortMultipartUpload**:
  Supports the `x-amz-if-match-initiated-time` header, which is true only if the multipart upload’s initialization time matches.

* **DeleteObject**:
  Adds support for:

  * `If-Match` (ETag comparison)
  * `x-amz-if-match-last-modified-time` (LastModified comparison)
  * `x-amz-if-match-size` (object size comparison)

Additionally, this PR updates precondition date parsing logic to support both **RFC1123** and **RFC3339** formats. Dates set in the future are ignored, matching AWS S3 behavior.
This commit is contained in:
niksis02
2025-09-09 01:55:38 +04:00
parent 04fbe405ca
commit 7a098b925f
12 changed files with 1111 additions and 160 deletions
+116 -6
View File
@@ -301,6 +301,11 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s
return s3response.PutObjectOutput{}, err
}
err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch)
if err != nil {
return s3response.PutObjectOutput{}, err
}
metadata := parseMetadata(po.Metadata)
// Store the "Expires" property in the object metadata
@@ -850,6 +855,42 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
}
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil {
// evaluate the preconditions before deleting the object
props, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: input.Bucket,
Key: input.Key,
})
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
// if object doesn't exist, skip preconditions
// if unexpected error shows up, return the error
return nil, err
}
if err == nil {
var etag string
if props.ETag != nil {
etag = *props.ETag
}
var lastMod time.Time
if props.LastModified != nil {
lastMod = *props.LastModified
}
var size int64
if props.ContentLength != nil {
size = *props.ContentLength
}
err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size,
backend.ObjectDeletePreconditions{
IfMatch: input.IfMatch,
IfMatchLastModTime: input.IfMatchLastModifiedTime,
IfMatchSize: input.IfMatchSize,
})
if err != nil {
return nil, err
}
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
if err != nil {
azerr, ok := err.(*azcore.ResponseError)
@@ -899,6 +940,26 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
if err != nil {
return s3response.CopyObjectOutput{}, err
}
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) {
_, err = az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &srcBucket,
Key: &srcObj,
IfMatch: input.CopySourceIfMatch,
IfNoneMatch: input.CopySourceIfNoneMatch,
IfModifiedSince: input.CopySourceIfModifiedSince,
IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
}
if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
if input.MetadataDirective != types.MetadataDirectiveReplace {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
@@ -977,11 +1038,6 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
}, nil
}
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
if err != nil {
return s3response.CopyObjectOutput{}, err
}
// Get the source object
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
if err != nil {
@@ -1210,7 +1266,7 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
if err != nil {
return s3response.CopyPartResult{}, nil
return s3response.CopyPartResult{}, err
}
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
@@ -1413,6 +1469,22 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
// Cleans up the initiated multipart upload in .sgwtmp namespace
func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
if input.IfMatchInitiatedTime != nil {
client, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
return err
}
resp, err := client.GetProperties(ctx, nil)
if err != nil {
return azureErrToS3Err(err)
}
if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
}
_, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil)
if err != nil {
return parseMpError(err)
@@ -1440,6 +1512,11 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip
func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
var res s3response.CompleteMultipartUploadResult
err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch)
if err != nil {
return s3response.CompleteMultipartUploadResult{}, "", err
}
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
if err != nil {
@@ -2058,6 +2135,29 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
return nil
}
func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error {
if areNils(ifMatch, ifNoneMatch) {
return nil
}
// call HeadObject to evaluate preconditions
// if object doesn't exist, move forward with the object creation
// otherwise return the error
_, err := az.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: bucket,
Key: object,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
if errors.Is(err, s3err.GetAPIError(s3err.ErrNotModified)) {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
return err
}
return nil
}
func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
data, ok := meta[string(key)]
if !ok {
@@ -2105,3 +2205,13 @@ func convertAzureEtag(etag *azcore.ETag) string {
return *backend.TrimEtag(str) + "-1"
}
func areNils[T any](args ...*T) bool {
for _, arg := range args {
if arg != nil {
return false
}
}
return true
}
+38
View File
@@ -532,3 +532,41 @@ func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreCond
return nil
}
// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions
func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error {
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
if ifNoneMatch != nil && *ifNoneMatch == etag {
return errPreconditionFailed
}
return nil
}
type ObjectDeletePreconditions struct {
IfMatch *string
IfMatchLastModTime *time.Time
IfMatchSize *int64
}
// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject
func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error {
ifMatch := preconditions.IfMatch
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
ifMatchTime := preconditions.IfMatchLastModTime
if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() {
return errPreconditionFailed
}
ifMatchSize := preconditions.IfMatchSize
if ifMatchSize != nil && *ifMatchSize != size {
return errPreconditionFailed
}
return nil
}
+110 -10
View File
@@ -1401,6 +1401,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
return res, "", err
}
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
if err == nil {
err = backend.EvaluateMatchPreconditions(string(b), input.IfMatch, input.IfNoneMatch)
if err != nil {
return res, "", err
}
}
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
@@ -2009,11 +2017,17 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err = os.Stat(filepath.Join(objdir, uploadID))
f, err := os.Stat(filepath.Join(objdir, uploadID))
if err != nil {
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if mpu.IfMatchInitiatedTime != nil {
if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() {
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
}
}
err = os.RemoveAll(filepath.Join(objdir, uploadID))
if err != nil {
return fmt.Errorf("remove multipart upload container: %w", err)
@@ -2600,6 +2614,32 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
return s3response.CopyPartResult{}, err
}
srcf, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
}
defer srcf.Close()
// evaluate preconditions
b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey)
srcEtag := string(b)
if err != nil {
srcEtag = ""
}
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
IfMatch: upi.CopySourceIfMatch,
IfNoneMatch: upi.CopySourceIfNoneMatch,
IfModSince: upi.CopySourceIfModifiedSince,
IfUnmodeSince: upi.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyPartResult{}, err
}
f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
*upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile)
if err != nil {
@@ -2610,15 +2650,6 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
defer f.cleanup()
srcf, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
}
defer srcf.Close()
rdr := io.NewSectionReader(srcf, startOffset, length)
hash := md5.New()
tr := io.TeeReader(rdr, hash)
@@ -2748,6 +2779,15 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3
name := filepath.Join(*po.Bucket, *po.Key)
// evaluate preconditions
etagBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, etagkey)
if err == nil {
err := backend.EvaluateMatchPreconditions(string(etagBytes), po.IfMatch, po.IfNoneMatch)
if err != nil {
return s3response.PutObjectOutput{}, err
}
}
uid, gid, doChown := p.getChownIDs(acct)
contentLength := int64(0)
@@ -3073,6 +3113,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, err
}
evalPreconditions := func(f os.FileInfo, bucket, object string) error {
var err error
if f == nil {
f, err = os.Stat(filepath.Join(bucket, object))
if err != nil {
return nil
}
}
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
etag := string(b)
if err != nil {
etag = ""
}
// evaluate preconditions
return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(),
backend.ObjectDeletePreconditions{
IfMatch: input.IfMatch,
IfMatchLastModTime: input.IfMatchLastModifiedTime,
IfMatchSize: input.IfMatchSize,
})
}
// Directory objects can't have versions
if !isDir && p.versioningEnabled() && vStatus != "" {
if getString(input.VersionId) == "" {
@@ -3089,6 +3153,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
err = evalPreconditions(fi, bucket, object)
if err != nil {
return nil, err
}
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
acct = auth.Account{}
@@ -3148,6 +3217,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}
if string(vId) == *input.VersionId {
// evaluate preconditions
err := evalPreconditions(nil, bucket, object)
if err != nil {
return nil, err
}
// if the specified VersionId is the same as in the latest version,
// remove the latest version, find the latest version from the versioning
// directory and move to the place of the deleted object, to make it the latest
@@ -3242,6 +3316,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}, nil
}
err = evalPreconditions(nil, versionPath, *input.VersionId)
if err != nil {
return nil, err
}
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
@@ -3289,6 +3368,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
return &s3.DeleteObjectOutput{}, nil
}
err = evalPreconditions(fi, bucket, object)
if err != nil {
return nil, err
}
err = os.Remove(objpath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
@@ -4031,6 +4115,22 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey)
srcEtag := string(b)
if err != nil {
srcEtag = ""
}
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
IfMatch: input.CopySourceIfMatch,
IfNoneMatch: input.CopySourceIfNoneMatch,
IfModSince: input.CopySourceIfModifiedSince,
IfUnmodeSince: input.CopySourceIfUnmodifiedSince,
})
if err != nil {
return s3response.CopyObjectOutput{}, err
}
mdmap := make(map[string]string)
p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap)
+14 -6
View File
@@ -69,6 +69,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
uploadId := ctx.Query("uploadId")
ifMatchInitiatedTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Initiated-Time"))
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
@@ -96,9 +97,10 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
err = c.be.AbortMultipartUpload(ctx.Context(),
&s3.AbortMultipartUploadInput{
UploadId: &uploadId,
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Bucket: &bucket,
Key: &key,
IfMatchInitiatedTime: ifMatchInitiatedTime,
})
return &Response{
MetaOpts: &MetaOptions{
@@ -113,6 +115,9 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
versionId := ctx.Query("versionId")
bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")
ifMatch := utils.GetStringPtr(ctx.Get("If-Match"))
ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time"))
ifMatchSize := utils.ParseIfMatchSize(ctx)
// context locals
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
@@ -165,9 +170,12 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
res, err := c.be.DeleteObject(ctx.Context(),
&s3.DeleteObjectInput{
Bucket: &bucket,
Key: &key,
VersionId: &versionId,
Bucket: &bucket,
Key: &key,
VersionId: &versionId,
IfMatch: ifMatch,
IfMatchLastModifiedTime: ifMatchLastModTime,
IfMatchSize: ifMatchSize,
})
if err != nil {
return &Response{
+4
View File
@@ -323,6 +323,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
}, err
}
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
@@ -338,6 +340,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumType: checksumType,
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
return &Response{
Data: res,
+22 -44
View File
@@ -21,7 +21,6 @@ import (
"io"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -355,14 +354,20 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) {
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
}
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
resp, err := c.be.UploadPartCopy(ctx.Context(),
&s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &key,
CopySource: &copySource,
PartNumber: &partNumber,
UploadId: &uploadId,
CopySourceRange: &copySrcRange,
Bucket: &bucket,
Key: &key,
CopySource: &copySource,
PartNumber: &partNumber,
UploadId: &uploadId,
CopySourceRange: &copySrcRange,
CopySourceIfMatch: preconditionHdrs.IfMatch,
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
})
var headers map[string]*string
if err == nil && resp.CopySourceVersionId != "" {
@@ -434,10 +439,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
bucket := ctx.Params("bucket")
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/")
copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match")
copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match")
copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since")
copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since")
metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy)))
taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy)))
contentType := ctx.Get("Content-Type")
@@ -480,35 +481,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
}, err
}
var mtime *time.Time
if copySrcModifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcModifSince)
if err != nil {
debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err)
// TODO: check the error type for invalid values
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
mtime = &tm
}
var umtime *time.Time
if copySrcUnmodifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
if err != nil {
debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err)
// TODO: check the error type for invalid values
return &Response{
MetaOpts: &MetaOptions{
BucketOwner: parsedAcl.Owner,
},
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
umtime = &tm
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace {
@@ -548,6 +520,8 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
}, err
}
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
res, err := c.be.CopyObject(ctx.Context(),
s3response.CopyObjectInput{
Bucket: &bucket,
@@ -561,10 +535,10 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
Tagging: &tagging,
TaggingDirective: taggingDirective,
CopySource: &copySource,
CopySourceIfMatch: &copySrcIfMatch,
CopySourceIfNoneMatch: &copySrcIfNoneMatch,
CopySourceIfModifiedSince: mtime,
CopySourceIfUnmodifiedSince: umtime,
CopySourceIfMatch: preconditionHdrs.IfMatch,
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
ExpectedBucketOwner: &acct.Access,
Metadata: metadata,
MetadataDirective: metaDirective,
@@ -691,6 +665,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
body = bytes.NewReader([]byte{})
}
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
res, err := c.be.PutObject(ctx.Context(),
s3response.PutObjectInput{
Bucket: &bucket,
@@ -714,6 +690,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
})
return &Response{
Headers: map[string]*string{
-36
View File
@@ -836,42 +836,6 @@ func TestS3ApiController_CopyObject(t *testing.T) {
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid copy source modified since",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Copy-Source": "bucket/object",
"X-Amz-Copy-Source-If-Modified-Since": "invalid_date",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid copy source unmodified since",
input: testInput{
locals: defaultLocals,
headers: map[string]string{
"X-Amz-Copy-Source": "bucket/object",
"X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date",
},
},
output: testOutput{
response: &Response{
MetaOpts: &MetaOptions{
BucketOwner: "root",
},
},
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
},
},
{
name: "invalid metadata directive",
input: testInput{
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"strconv"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/debuglogger"
)
// ConditionalHeaders holds the conditional header values
type ConditionalHeaders struct {
IfMatch *string
IfNoneMatch *string
IfModSince *time.Time
IfUnmodeSince *time.Time
}
type precondtionCfg struct {
withCopySource bool
}
type preconditionOpt func(*precondtionCfg)
func WithCopySource() preconditionOpt {
return func(o *precondtionCfg) { o.withCopySource = true }
}
// ParsePreconditionHeaders parses the precondition headers:
// - If-Match
// - If-None-Match
// - If-Modified-Since
// - If-Unmodified-Since
func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders {
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...)
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...)
return ConditionalHeaders{
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
IfModSince: ifModSince,
IfUnmodeSince: ifUnmodeSince,
}
}
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
}
prefix := ""
if cfg.withCopySource {
prefix = "X-Amz-Copy-Source-"
}
return GetStringPtr(ctx.Get(prefix + "If-Match")), GetStringPtr(ctx.Get(prefix + "If-None-Match"))
}
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
// headers from fiber context to *time.Time
func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) {
cfg := new(precondtionCfg)
for _, opt := range opts {
opt(cfg)
}
prefix := ""
if cfg.withCopySource {
prefix = "X-Amz-Copy-Source-"
}
ifModSince := ctx.Get(prefix + "If-Modified-Since")
ifUnmodSince := ctx.Get(prefix + "If-Unmodified-Since")
ifModSinceParsed := ParsePreconditionDateHeader(ifModSince)
ifUnmodSinceParsed := ParsePreconditionDateHeader(ifUnmodSince)
return ifModSinceParsed, ifUnmodSinceParsed
}
// ParsePreconditionDateHeader tries to parse the given date string as
// - RFC1123
// - RFC3339
// both are valid
func ParsePreconditionDateHeader(date string) *time.Time {
if date == "" {
return nil
}
// try to parse as RFC1123
parsed, err := time.Parse(time.RFC1123, date)
if err == nil {
// ignore future dates
if parsed.After(time.Now()) {
return nil
}
return &parsed
}
// try to parse as RFC3339
parsed, err = time.Parse(time.RFC3339, date)
if err == nil {
// ignore future dates
if parsed.After(time.Now()) {
return nil
}
return &parsed
}
return nil
}
// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64
// if parsing fails, returns nil
func ParseIfMatchSize(ctx *fiber.Ctx) *int64 {
ifMatchSizeHdr := ctx.Get("x-amz-if-match-size")
ifMatchSize, err := strconv.ParseInt(ifMatchSizeHdr, 10, 64)
if err != nil {
debuglogger.Logf("failed to parse 'x-amz-if-match-size': %s", ifMatchSizeHdr)
return nil
}
return &ifMatchSize
}
-53
View File
@@ -642,59 +642,6 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type
return algo, chType, nil
}
// ConditionalHeaders holds the conditional header values
type ConditionalHeaders struct {
IfMatch *string
IfNoneMatch *string
IfModSince *time.Time
IfUnmodeSince *time.Time
}
// ParsePreconditionHeaders parses the precondition headers:
// - If-Match
// - If-None-Match
// - If-Modified-Since
// - If-Unmodified-Since
func ParsePreconditionHeaders(ctx *fiber.Ctx) ConditionalHeaders {
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx)
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx)
return ConditionalHeaders{
IfMatch: ifMatch,
IfNoneMatch: ifNoneMatch,
IfModSince: ifModSince,
IfUnmodeSince: ifUnmodeSince,
}
}
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx) (*string, *string) {
return GetStringPtr(ctx.Get("If-Match")), GetStringPtr(ctx.Get("If-None-Match"))
}
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
// headers from fiber context to *time.Time
func ParsePreconditionDateHeaders(ctx *fiber.Ctx) (*time.Time, *time.Time) {
ifModSince := ctx.Get("If-Modified-Since")
ifUnmodSince := ctx.Get("If-Unmodified-Since")
var ifModSinceParsed, ifUnmodSinceParsed *time.Time
// the time format should be a valid RFC1123
// if parsing fails, ignore the error and leave the value as nil
modParsed, err := time.Parse(time.RFC1123, ifModSince)
if err == nil {
ifModSinceParsed = &modParsed
}
unmodParsed, err := time.Parse(time.RFC1123, ifUnmodSince)
if err == nil {
ifUnmodSinceParsed = &unmodParsed
}
return ifModSinceParsed, ifUnmodSinceParsed
}
// TagLimit specifies the allowed tag count in a tag set
type TagLimit int
+12
View File
@@ -149,6 +149,7 @@ func TestPutObject(s *S3Conf) {
PutObject_with_object_lock(s)
PutObject_invalid_legal_hold(s)
PutObject_invalid_object_lock_mode(s)
PutObject_conditional_writes(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
PutObject_checksum_algorithm_and_header_mismatch(s)
@@ -279,6 +280,7 @@ func TestDeleteObject(s *S3Conf) {
DeleteObject_non_existing_dir_object(s)
DeleteObject_directory_object(s)
DeleteObject_non_empty_dir_obj(s)
DeleteObject_conditional_writes(s)
DeleteObject_success(s)
DeleteObject_success_status_code(s)
}
@@ -307,6 +309,7 @@ func TestCopyObject(s *S3Conf) {
CopyObject_invalid_object_lock_mode(s)
CopyObject_with_legal_hold(s)
CopyObject_with_retention_lock(s)
CopyObject_conditional_reads(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
CopyObject_invalid_checksum_algorithm(s)
@@ -398,6 +401,7 @@ func TestUploadPartCopy(s *S3Conf) {
UploadPartCopy_should_copy_the_checksum(s)
UploadPartCopy_should_not_copy_the_checksum(s)
UploadPartCopy_should_calculate_the_checksum(s)
UploadPartCopy_conditional_reads(s)
}
}
@@ -437,6 +441,7 @@ func TestAbortMultipartUpload(s *S3Conf) {
AbortMultipartUpload_incorrect_object_key(s)
AbortMultipartUpload_success(s)
AbortMultipartUpload_success_status_code(s)
AbortMultipartUpload_if_match_initiated_time(s)
}
func TestCompleteMultipartUpload(s *S3Conf) {
@@ -448,6 +453,7 @@ func TestCompleteMultipartUpload(s *S3Conf) {
CompleteMultipartUpload_empty_parts(s)
CompleteMultipartUpload_incorrect_parts_order(s)
CompleteMultipartUpload_mpu_object_size(s)
CompleteMultipartUpload_conditional_writes(s)
//TODO: remove the condition after implementing checksums in azure
if !s.azureTests {
CompleteMultipartUpload_invalid_checksum_type(s)
@@ -1053,6 +1059,7 @@ func GetIntTests() IntTests {
"PutObject_with_object_lock": PutObject_with_object_lock,
"PutObject_invalid_legal_hold": PutObject_invalid_legal_hold,
"PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode,
"PutObject_conditional_writes": PutObject_conditional_writes,
"PutObject_invalid_credentials": PutObject_invalid_credentials,
"PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch,
"PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers,
@@ -1184,6 +1191,7 @@ func GetIntTests() IntTests {
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
"DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash,
"DeleteObject_non_empty_dir_obj": DeleteObject_non_empty_dir_obj,
"DeleteObject_conditional_writes": DeleteObject_conditional_writes,
"DeleteObject_name_too_long": DeleteObject_name_too_long,
"CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object,
"CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object,
@@ -1211,6 +1219,7 @@ func GetIntTests() IntTests {
"CopyObject_invalid_object_lock_mode": CopyObject_invalid_object_lock_mode,
"CopyObject_with_legal_hold": CopyObject_with_legal_hold,
"CopyObject_with_retention_lock": CopyObject_with_retention_lock,
"CopyObject_conditional_reads": CopyObject_conditional_reads,
"CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm,
"CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy,
"CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum,
@@ -1270,6 +1279,7 @@ func GetIntTests() IntTests {
"UploadPartCopy_exceeding_copy_source_range": UploadPartCopy_exceeding_copy_source_range,
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
"UploadPartCopy_conditional_reads": UploadPartCopy_conditional_reads,
"UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum,
"UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum,
"UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum,
@@ -1295,6 +1305,7 @@ func GetIntTests() IntTests {
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
"AbortMultipartUpload_success": AbortMultipartUpload_success,
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
"AbortMultipartUpload_if_match_initiated_time": AbortMultipartUpload_if_match_initiated_time,
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
@@ -1302,6 +1313,7 @@ func GetIntTests() IntTests {
"CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts,
"CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order,
"CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size,
"CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes,
"CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type,
"CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part,
"CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part,
+648
View File
@@ -3043,6 +3043,7 @@ func PutObject_invalid_legal_hold(s *S3Conf) error {
return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus))
}, withLock())
}
func PutObject_invalid_object_lock_mode(s *S3Conf) error {
testName := "PutObject_invalid_object_lock_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -3057,6 +3058,78 @@ func PutObject_invalid_object_lock_mode(s *S3Conf) error {
}, withLock())
}
func PutObject_conditional_writes(s *S3Conf) error {
testName := "PutObject_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
etag := res.res.ETag
incorrectEtag := getPtr("incorrect_etag")
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
obj string
ifMatch *string
ifNoneMatch *string
err error
}{
{obj, etag, nil, nil},
{obj, etag, etag, errPrecond},
{obj, etag, incorrectEtag, nil},
{obj, incorrectEtag, incorrectEtag, errPrecond},
{obj, incorrectEtag, etag, errPrecond},
{obj, incorrectEtag, nil, errPrecond},
{obj, nil, incorrectEtag, nil},
{obj, nil, etag, errPrecond},
{obj, nil, nil, nil},
// should ignore the precondition headers if
// an object with the given name doesn't exist
{"obj-1", incorrectEtag, etag, nil},
{"obj-2", etag, etag, nil},
{"obj-3", etag, incorrectEtag, nil},
{"obj-4", incorrectEtag, nil, nil},
{"obj-5", nil, etag, nil},
} {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &test.obj,
Body: bytes.NewReader([]byte("dummy")),
IfMatch: test.ifMatch,
IfNoneMatch: test.ifNoneMatch,
}, s3client)
if err == nil {
// azure blob storage generates different ETags for
// the exact same data.
// to avoid ETag collision reassign the etag value
*etag = *res.res.ETag
}
if test.err == nil && err != nil {
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
}
if test.err != nil {
apierr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
}
if err := checkApiErr(err, apierr); err != nil {
return fmt.Errorf("test case %v: %w", i, err)
}
}
}
return nil
})
}
func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error {
testName := "PutObject_checksum_algorithm_and_header_mismatch"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -6895,6 +6968,98 @@ func DeleteObject_non_empty_dir_obj(s *S3Conf) error {
})
}
func DeleteObject_conditional_writes(s *S3Conf) error {
testName := "DeleteObject_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
var etag *string = getPtr("")
var size *int64 = getPtr(int64(0))
var modTime *time.Time = getPtr(time.Now())
createObj := func() error {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
// get the exact LastModified time
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
*etag = *res.res.ETag
*size = *res.res.Size
*modTime = *out.LastModified
return nil
}
err := createObj()
if err != nil {
return err
}
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
ifMatch *string
size *int64
modTime *time.Time
err error
}{
// no error cases
{etag, size, modTime, nil},
{etag, nil, nil, nil},
{nil, size, nil, nil},
{nil, nil, modTime, nil},
{etag, size, nil, nil},
{etag, nil, modTime, nil},
{nil, size, modTime, nil},
// error cases
{getPtr("incorrect_etag"), nil, nil, errPrecond},
{nil, getPtr(int64(23234)), nil, errPrecond},
{nil, nil, getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
{getPtr("incorrect_etag"), getPtr(int64(23234)), nil, errPrecond},
{getPtr("incorrect_etag"), getPtr(int64(23234)), getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
} {
err := createObj()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: &bucket,
Key: &obj,
IfMatch: test.ifMatch,
IfMatchSize: test.size,
IfMatchLastModifiedTime: test.modTime,
})
cancel()
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func DeleteObject_directory_not_empty(s *S3Conf) error {
testName := "DeleteObject_directory_not_empty"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -8022,6 +8187,156 @@ func CopyObject_with_retention_lock(s *S3Conf) error {
}, withLock())
}
func CopyObject_conditional_reads(s *S3Conf) error {
testName := "CopyObject_conditional_reads"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
key := "my-obj"
obj, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
}, s3client)
if err != nil {
return err
}
errMod := s3err.GetAPIError(s3err.ErrNotModified)
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
// sleep one second to get dates before and after
// the object creation
time.Sleep(time.Second * 1)
before := time.Now().AddDate(0, 0, -3)
after := time.Now()
etag := obj.res.ETag
for i, test := range []struct {
ifmatch *string
ifnonematch *string
ifmodifiedsince *time.Time
ifunmodifiedsince *time.Time
err error
}{
// all the cases when preconditions are either empty, true or false
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
{getPtr("invalid_etag"), etag, &before, &before, errCond},
{getPtr("invalid_etag"), etag, &before, &after, errCond},
{getPtr("invalid_etag"), etag, &before, nil, errCond},
{getPtr("invalid_etag"), etag, &after, &before, errCond},
{getPtr("invalid_etag"), etag, &after, &after, errCond},
{getPtr("invalid_etag"), etag, &after, nil, errCond},
{getPtr("invalid_etag"), etag, nil, &before, errCond},
{getPtr("invalid_etag"), etag, nil, &after, errCond},
{getPtr("invalid_etag"), etag, nil, nil, errCond},
{getPtr("invalid_etag"), nil, &before, &before, errCond},
{getPtr("invalid_etag"), nil, &before, &after, errCond},
{getPtr("invalid_etag"), nil, &before, nil, errCond},
{getPtr("invalid_etag"), nil, &after, &before, errCond},
{getPtr("invalid_etag"), nil, &after, &after, errCond},
{getPtr("invalid_etag"), nil, &after, nil, errCond},
{getPtr("invalid_etag"), nil, nil, &before, errCond},
{getPtr("invalid_etag"), nil, nil, &after, errCond},
{getPtr("invalid_etag"), nil, nil, nil, errCond},
{etag, getPtr("invalid_etag"), &before, &before, nil},
{etag, getPtr("invalid_etag"), &before, &after, nil},
{etag, getPtr("invalid_etag"), &before, nil, nil},
{etag, getPtr("invalid_etag"), &after, &before, nil},
{etag, getPtr("invalid_etag"), &after, &after, nil},
{etag, getPtr("invalid_etag"), &after, nil, nil},
{etag, getPtr("invalid_etag"), nil, &before, nil},
{etag, getPtr("invalid_etag"), nil, &after, nil},
{etag, getPtr("invalid_etag"), nil, nil, nil},
{etag, etag, &before, &before, errMod},
{etag, etag, &before, &after, errMod},
{etag, etag, &before, nil, errMod},
{etag, etag, &after, &before, errMod},
{etag, etag, &after, &after, errMod},
{etag, etag, &after, nil, errMod},
{etag, etag, nil, &before, errMod},
{etag, etag, nil, &after, errMod},
{etag, etag, nil, nil, errMod},
{etag, nil, &before, &before, nil},
{etag, nil, &before, &after, nil},
{etag, nil, &before, nil, nil},
{etag, nil, &after, &before, errMod},
{etag, nil, &after, &after, errMod},
{etag, nil, &after, nil, errMod},
{etag, nil, nil, &before, nil},
{etag, nil, nil, &after, nil},
{etag, nil, nil, nil, nil},
{nil, getPtr("invalid_etag"), &before, &before, errCond},
{nil, getPtr("invalid_etag"), &before, &after, nil},
{nil, getPtr("invalid_etag"), &before, nil, nil},
{nil, getPtr("invalid_etag"), &after, &before, errCond},
{nil, getPtr("invalid_etag"), &after, &after, nil},
{nil, getPtr("invalid_etag"), &after, nil, nil},
{nil, getPtr("invalid_etag"), nil, &before, errCond},
{nil, getPtr("invalid_etag"), nil, &after, nil},
{nil, getPtr("invalid_etag"), nil, nil, nil},
{nil, etag, &before, &before, errCond},
{nil, etag, &before, &after, errMod},
{nil, etag, &before, nil, errMod},
{nil, etag, &after, &before, errCond},
{nil, etag, &after, &after, errMod},
{nil, etag, &after, nil, errMod},
{nil, etag, nil, &before, errCond},
{nil, etag, nil, &after, errMod},
{nil, etag, nil, nil, errMod},
{nil, nil, &before, &before, errCond},
{nil, nil, &before, &after, nil},
{nil, nil, &before, nil, nil},
{nil, nil, &after, &before, errCond},
{nil, nil, &after, &after, errMod},
{nil, nil, &after, nil, errMod},
{nil, nil, nil, &before, errCond},
{nil, nil, nil, &after, nil},
{nil, nil, nil, nil, nil},
} {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: getPtr("dst-obj"),
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
CopySourceIfMatch: test.ifmatch,
CopySourceIfNoneMatch: test.ifnonematch,
CopySourceIfModifiedSince: test.ifmodifiedsince,
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func CopyObject_invalid_checksum_algorithm(s *S3Conf) error {
testName := "CopyObject_invalid_checksum_algorithm"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -10476,6 +10791,164 @@ func UploadPartCopy_by_range_success(s *S3Conf) error {
})
}
func UploadPartCopy_conditional_reads(s *S3Conf) error {
testName := "UploadPartCopy_conditional_reads"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
key := "my-obj"
obj, err := putObjectWithData(10, &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
}, s3client)
if err != nil {
return err
}
errMod := s3err.GetAPIError(s3err.ErrNotModified)
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
// sleep one second to get dates before and after
// the object creation
time.Sleep(time.Second * 1)
before := time.Now().AddDate(0, 0, -3)
after := time.Now()
etag := obj.res.ETag
for i, test := range []struct {
ifmatch *string
ifnonematch *string
ifmodifiedsince *time.Time
ifunmodifiedsince *time.Time
err error
}{
// all the cases when preconditions are either empty, true or false
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
{getPtr("invalid_etag"), etag, &before, &before, errCond},
{getPtr("invalid_etag"), etag, &before, &after, errCond},
{getPtr("invalid_etag"), etag, &before, nil, errCond},
{getPtr("invalid_etag"), etag, &after, &before, errCond},
{getPtr("invalid_etag"), etag, &after, &after, errCond},
{getPtr("invalid_etag"), etag, &after, nil, errCond},
{getPtr("invalid_etag"), etag, nil, &before, errCond},
{getPtr("invalid_etag"), etag, nil, &after, errCond},
{getPtr("invalid_etag"), etag, nil, nil, errCond},
{getPtr("invalid_etag"), nil, &before, &before, errCond},
{getPtr("invalid_etag"), nil, &before, &after, errCond},
{getPtr("invalid_etag"), nil, &before, nil, errCond},
{getPtr("invalid_etag"), nil, &after, &before, errCond},
{getPtr("invalid_etag"), nil, &after, &after, errCond},
{getPtr("invalid_etag"), nil, &after, nil, errCond},
{getPtr("invalid_etag"), nil, nil, &before, errCond},
{getPtr("invalid_etag"), nil, nil, &after, errCond},
{getPtr("invalid_etag"), nil, nil, nil, errCond},
{etag, getPtr("invalid_etag"), &before, &before, nil},
{etag, getPtr("invalid_etag"), &before, &after, nil},
{etag, getPtr("invalid_etag"), &before, nil, nil},
{etag, getPtr("invalid_etag"), &after, &before, nil},
{etag, getPtr("invalid_etag"), &after, &after, nil},
{etag, getPtr("invalid_etag"), &after, nil, nil},
{etag, getPtr("invalid_etag"), nil, &before, nil},
{etag, getPtr("invalid_etag"), nil, &after, nil},
{etag, getPtr("invalid_etag"), nil, nil, nil},
{etag, etag, &before, &before, errMod},
{etag, etag, &before, &after, errMod},
{etag, etag, &before, nil, errMod},
{etag, etag, &after, &before, errMod},
{etag, etag, &after, &after, errMod},
{etag, etag, &after, nil, errMod},
{etag, etag, nil, &before, errMod},
{etag, etag, nil, &after, errMod},
{etag, etag, nil, nil, errMod},
{etag, nil, &before, &before, nil},
{etag, nil, &before, &after, nil},
{etag, nil, &before, nil, nil},
{etag, nil, &after, &before, errMod},
{etag, nil, &after, &after, errMod},
{etag, nil, &after, nil, errMod},
{etag, nil, nil, &before, nil},
{etag, nil, nil, &after, nil},
{etag, nil, nil, nil, nil},
{nil, getPtr("invalid_etag"), &before, &before, errCond},
{nil, getPtr("invalid_etag"), &before, &after, nil},
{nil, getPtr("invalid_etag"), &before, nil, nil},
{nil, getPtr("invalid_etag"), &after, &before, errCond},
{nil, getPtr("invalid_etag"), &after, &after, nil},
{nil, getPtr("invalid_etag"), &after, nil, nil},
{nil, getPtr("invalid_etag"), nil, &before, errCond},
{nil, getPtr("invalid_etag"), nil, &after, nil},
{nil, getPtr("invalid_etag"), nil, nil, nil},
{nil, etag, &before, &before, errCond},
{nil, etag, &before, &after, errMod},
{nil, etag, &before, nil, errMod},
{nil, etag, &after, &before, errCond},
{nil, etag, &after, &after, errMod},
{nil, etag, &after, nil, errMod},
{nil, etag, nil, &before, errCond},
{nil, etag, nil, &after, errMod},
{nil, etag, nil, nil, errMod},
{nil, nil, &before, &before, errCond},
{nil, nil, &before, &after, nil},
{nil, nil, &before, nil, nil},
{nil, nil, &after, &before, errCond},
{nil, nil, &after, &after, errMod},
{nil, nil, &after, nil, errMod},
{nil, nil, nil, &before, errCond},
{nil, nil, nil, &after, nil},
{nil, nil, nil, nil, nil},
} {
mpKey := "mp-key"
mp, err := createMp(s3client, bucket, mpKey)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
Bucket: &bucket,
Key: &mpKey,
UploadId: mp.UploadId,
PartNumber: getPtr(int32(1)),
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
CopySourceIfMatch: test.ifmatch,
CopySourceIfNoneMatch: test.ifnonematch,
CopySourceIfModifiedSince: test.ifmodifiedsince,
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
testName := "UploadPartCopy_should_copy_the_checksum"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -11427,6 +11900,87 @@ func AbortMultipartUpload_success_status_code(s *S3Conf) error {
})
}
func AbortMultipartUpload_if_match_initiated_time(s *S3Conf) error {
testName := "AbortMultipartUpload_if_match_initiated_time"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
var initiated *time.Time = getPtr(time.Now())
// createMpUpload creates a multipart uplod
// and retruns the uploadId and creation date
abortMp := func(date *time.Time) error {
mpObj := "my-obj"
mp, err := createMp(s3client, bucket, mpObj)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
var initiatedTime *time.Time
for _, up := range res.Uploads {
if getString(up.UploadId) == getString(mp.UploadId) {
initiatedTime = up.Initiated
break
}
}
if initiatedTime == nil {
return fmt.Errorf("unexpected err: the multipart upload is not found")
}
*initiated = *initiatedTime
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: &bucket,
Key: &mpObj,
UploadId: mp.UploadId,
IfMatchInitiatedTime: date,
})
cancel()
return err
}
for i, test := range []struct {
date *time.Time
err error
}{
{nil, nil},
// match: success case
{initiated, nil},
// should ignore future dates
{getPtr(initiated.AddDate(1, 0, 0)), nil},
// should fail if the initation date doesn't match
{getPtr(initiated.AddDate(-1, 0, 1)), s3err.GetAPIError(s3err.ErrPreconditionFailed)},
} {
err := abortMp(test.date)
if test.err == nil && err != nil {
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
}
if test.err != nil {
apiErr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("invalid error type: expected s3err.APIError")
}
if err := checkApiErr(err, apiErr); err != nil {
return fmt.Errorf("test case %d failed: %w", i, err)
}
}
}
return nil
})
}
func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error {
testName := "CompletedMultipartUpload_non_existing_bucket"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -12593,6 +13147,100 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error {
})
}
func CompleteMultipartUpload_conditional_writes(s *S3Conf) error {
testName := "CompleteMultipartUpload_conditional_writes"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
etag := getPtr("")
incorrectEtag := getPtr("incorrect_etag")
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
for i, test := range []struct {
obj string
ifMatch *string
ifNoneMatch *string
err error
}{
{obj, etag, nil, nil},
{obj, etag, etag, errPrecond},
{obj, etag, incorrectEtag, nil},
{obj, incorrectEtag, incorrectEtag, errPrecond},
{obj, incorrectEtag, etag, errPrecond},
{obj, incorrectEtag, nil, errPrecond},
{obj, nil, incorrectEtag, nil},
{obj, nil, etag, errPrecond},
{obj, nil, nil, nil},
// should ignore the precondition headers if
// an object with the given name doesn't exist
{"obj-1", incorrectEtag, etag, nil},
{"obj-2", etag, etag, nil},
{"obj-3", etag, incorrectEtag, nil},
{"obj-4", incorrectEtag, nil, nil},
{"obj-5", nil, etag, nil},
} {
res, err := putObjectWithData(0, &s3.PutObjectInput{
Bucket: &bucket,
Key: &obj,
Body: bytes.NewReader([]byte("dummy")),
}, s3client)
if err != nil {
return err
}
// azure blob storage generates different ETags for
// the exact same data.
// to avoid ETag collision reassign the etag value
*etag = *res.res.ETag
mp, err := createMp(s3client, bucket, test.obj)
if err != nil {
return err
}
parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, test.obj, *mp.UploadId)
if err != nil {
return err
}
part := parts[0]
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &test.obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: []types.CompletedPart{
{
ETag: part.ETag,
PartNumber: getPtr(int32(1)),
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
},
},
},
IfMatch: test.ifMatch,
IfNoneMatch: test.ifNoneMatch,
})
cancel()
if test.err == nil && err != nil {
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
}
if test.err != nil {
apierr, ok := test.err.(s3err.APIError)
if !ok {
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
}
if err := checkApiErr(err, apierr); err != nil {
return fmt.Errorf("test case %v: %w", i, err)
}
}
}
return nil
})
}
func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error {
testName := "CompleteMultipartUpload_invalid_part_number"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+9 -5
View File
@@ -518,11 +518,15 @@ type putObjectOutput struct {
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
data := make([]byte, lgth)
rand.Read(data)
csum := sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
var csum [32]byte
var data []byte
if input.Body == nil {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := client.PutObject(ctx, input)