From 7a098b925faf3cc3d6e5db167e3028d5077c4f50 Mon Sep 17 00:00:00 2001 From: niksis02 Date: Mon, 8 Sep 2025 23:36:05 +0400 Subject: [PATCH] feat: implement conditional writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #821 **Implements conditional operations across object APIs:** * **PutObject** and **CompleteMultipartUpload**: Supports conditional writes with `If-Match` and `If-None-Match` headers (ETag comparisons). Evaluation is based on an existing object with the same key in the bucket. The operation is allowed only if the preconditions are satisfied. If no object exists for the key, these headers are ignored. * **CopyObject** and **UploadPartCopy**: Adds conditional reads on the copy source object with the following headers: * `x-amz-copy-source-if-match` * `x-amz-copy-source-if-none-match` * `x-amz-copy-source-if-modified-since` * `x-amz-copy-source-if-unmodified-since` The first two are ETag comparisons, while the latter two compare against the copy source’s `LastModified` timestamp. * **AbortMultipartUpload**: Supports the `x-amz-if-match-initiated-time` header, which is true only if the multipart upload’s initialization time matches. * **DeleteObject**: Adds support for: * `If-Match` (ETag comparison) * `x-amz-if-match-last-modified-time` (LastModified comparison) * `x-amz-if-match-size` (object size comparison) Additionally, this PR updates precondition date parsing logic to support both **RFC1123** and **RFC3339** formats. Dates set in the future are ignored, matching AWS S3 behavior. --- backend/azure/azure.go | 122 ++++- backend/common.go | 38 ++ backend/posix/posix.go | 120 ++++- s3api/controllers/object-delete.go | 20 +- s3api/controllers/object-post.go | 4 + s3api/controllers/object-put.go | 66 +-- s3api/controllers/object-put_test.go | 36 -- s3api/utils/precondition.go | 138 ++++++ s3api/utils/utils.go | 53 --- tests/integration/group-tests.go | 12 + tests/integration/tests.go | 648 +++++++++++++++++++++++++++ tests/integration/utils.go | 14 +- 12 files changed, 1111 insertions(+), 160 deletions(-) create mode 100644 s3api/utils/precondition.go diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 0c3286d2..5c2ec8d6 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -301,6 +301,11 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s return s3response.PutObjectOutput{}, err } + err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + metadata := parseMetadata(po.Metadata) // Store the "Expires" property in the object metadata @@ -850,6 +855,42 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input } func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil { + // evaluate the preconditions before deleting the object + props, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + }) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + // if object doesn't exist, skip preconditions + // if unexpected error shows up, return the error + return nil, err + } + if err == nil { + var etag string + if props.ETag != nil { + etag = *props.ETag + } + var lastMod time.Time + if props.LastModified != nil { + lastMod = *props.LastModified + } + var size int64 + if props.ContentLength != nil { + size = *props.ContentLength + } + err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size, + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + if err != nil { + return nil, err + } + } + } + _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) if err != nil { azerr, ok := err.(*azcore.ResponseError) @@ -899,6 +940,26 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu if err != nil { return s3response.CopyObjectOutput{}, err } + + srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) { + _, err = az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModifiedSince: input.CopySourceIfModifiedSince, + IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + } + if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource { if input.MetadataDirective != types.MetadataDirectiveReplace { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) @@ -977,11 +1038,6 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu }, nil } - srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource) - if err != nil { - return s3response.CopyObjectOutput{}, err - } - // Get the source object downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil) if err != nil { @@ -1210,7 +1266,7 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3 func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return s3response.CopyPartResult{}, nil + return s3response.CopyPartResult{}, err } if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { @@ -1413,6 +1469,22 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa // Cleans up the initiated multipart upload in .sgwtmp namespace func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) + + if input.IfMatchInitiatedTime != nil { + client, err := az.getBlobClient(*input.Bucket, tmpPath) + if err != nil { + return err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } _, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil) if err != nil { return parseMpError(err) @@ -1440,6 +1512,11 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { var res s3response.CompleteMultipartUploadResult + err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", err + } + tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) blobClient, err := az.getBlobClient(*input.Bucket, tmpPath) if err != nil { @@ -2058,6 +2135,29 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string return nil } +func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error { + if areNils(ifMatch, ifNoneMatch) { + return nil + } + // call HeadObject to evaluate preconditions + // if object doesn't exist, move forward with the object creation + // otherwise return the error + _, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: bucket, + Key: object, + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, + }) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNotModified)) { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return err + } + + return nil +} + func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) { data, ok := meta[string(key)] if !ok { @@ -2105,3 +2205,13 @@ func convertAzureEtag(etag *azcore.ETag) string { return *backend.TrimEtag(str) + "-1" } + +func areNils[T any](args ...*T) bool { + for _, arg := range args { + if arg != nil { + return false + } + } + + return true +} diff --git a/backend/common.go b/backend/common.go index fd3a68ce..f687c1e0 100644 --- a/backend/common.go +++ b/backend/common.go @@ -532,3 +532,41 @@ func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreCond return nil } + +// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions +func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error { + if ifMatch != nil && *ifMatch != etag { + return errPreconditionFailed + } + if ifNoneMatch != nil && *ifNoneMatch == etag { + return errPreconditionFailed + } + + return nil +} + +type ObjectDeletePreconditions struct { + IfMatch *string + IfMatchLastModTime *time.Time + IfMatchSize *int64 +} + +// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject +func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error { + ifMatch := preconditions.IfMatch + if ifMatch != nil && *ifMatch != etag { + return errPreconditionFailed + } + + ifMatchTime := preconditions.IfMatchLastModTime + if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() { + return errPreconditionFailed + } + + ifMatchSize := preconditions.IfMatchSize + if ifMatchSize != nil && *ifMatchSize != size { + return errPreconditionFailed + } + + return nil +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 91b9a9d6..0a4b9495 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1401,6 +1401,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return res, "", err } + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if err == nil { + err = backend.EvaluateMatchPreconditions(string(b), input.IfMatch, input.IfNoneMatch) + if err != nil { + return res, "", err + } + } + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) @@ -2009,11 +2017,17 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) - _, err = os.Stat(filepath.Join(objdir, uploadID)) + f, err := os.Stat(filepath.Join(objdir, uploadID)) if err != nil { return s3err.GetAPIError(s3err.ErrNoSuchUpload) } + if mpu.IfMatchInitiatedTime != nil { + if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } + err = os.RemoveAll(filepath.Join(objdir, uploadID)) if err != nil { return fmt.Errorf("remove multipart upload container: %w", err) @@ -2600,6 +2614,32 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) return s3response.CopyPartResult{}, err } + srcf, err := os.Open(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) + } + defer srcf.Close() + + // evaluate preconditions + b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil { + srcEtag = "" + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: upi.CopySourceIfMatch, + IfNoneMatch: upi.CopySourceIfNoneMatch, + IfModSince: upi.CopySourceIfModifiedSince, + IfUnmodeSince: upi.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyPartResult{}, err + } + f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir), *upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile) if err != nil { @@ -2610,15 +2650,6 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } defer f.cleanup() - srcf, err := os.Open(objPath) - if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil { - return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) - } - defer srcf.Close() - rdr := io.NewSectionReader(srcf, startOffset, length) hash := md5.New() tr := io.TeeReader(rdr, hash) @@ -2748,6 +2779,15 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3 name := filepath.Join(*po.Bucket, *po.Key) + // evaluate preconditions + etagBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, etagkey) + if err == nil { + err := backend.EvaluateMatchPreconditions(string(etagBytes), po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + uid, gid, doChown := p.getChownIDs(acct) contentLength := int64(0) @@ -3073,6 +3113,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, err } + evalPreconditions := func(f os.FileInfo, bucket, object string) error { + var err error + if f == nil { + f, err = os.Stat(filepath.Join(bucket, object)) + if err != nil { + return nil + } + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + etag := string(b) + if err != nil { + etag = "" + } + + // evaluate preconditions + return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(), + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + } + // Directory objects can't have versions if !isDir && p.versioningEnabled() && vStatus != "" { if getString(input.VersionId) == "" { @@ -3089,6 +3153,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} @@ -3148,6 +3217,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } if string(vId) == *input.VersionId { + // evaluate preconditions + err := evalPreconditions(nil, bucket, object) + if err != nil { + return nil, err + } // if the specified VersionId is the same as in the latest version, // remove the latest version, find the latest version from the versioning // directory and move to the place of the deleted object, to make it the latest @@ -3242,6 +3316,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }, nil } + err = evalPreconditions(nil, versionPath, *input.VersionId) + if err != nil { + return nil, err + } + isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId) err = os.Remove(filepath.Join(versionPath, *input.VersionId)) @@ -3289,6 +3368,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + err = os.Remove(objpath) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) @@ -4031,6 +4115,22 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } + b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil { + srcEtag = "" + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModSince: input.CopySourceIfModifiedSince, + IfUnmodeSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + mdmap := make(map[string]string) p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap) diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go index d625fe67..bfb5367d 100644 --- a/s3api/controllers/object-delete.go +++ b/s3api/controllers/object-delete.go @@ -69,6 +69,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) uploadId := ctx.Query("uploadId") + ifMatchInitiatedTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Initiated-Time")) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) @@ -96,9 +97,10 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) err = c.be.AbortMultipartUpload(ctx.Context(), &s3.AbortMultipartUploadInput{ - UploadId: &uploadId, - Bucket: &bucket, - Key: &key, + UploadId: &uploadId, + Bucket: &bucket, + Key: &key, + IfMatchInitiatedTime: ifMatchInitiatedTime, }) return &Response{ MetaOpts: &MetaOptions{ @@ -113,6 +115,9 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) versionId := ctx.Query("versionId") bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + ifMatch := utils.GetStringPtr(ctx.Get("If-Match")) + ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time")) + ifMatchSize := utils.ParseIfMatchSize(ctx) // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) @@ -165,9 +170,12 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { res, err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &key, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + VersionId: &versionId, + IfMatch: ifMatch, + IfMatchLastModifiedTime: ifMatchLastModTime, + IfMatchSize: ifMatchSize, }) if err != nil { return &Response{ diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 35614efd..4f4df452 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -323,6 +323,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err }, err } + ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx) + res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{ Bucket: &bucket, @@ -338,6 +340,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), ChecksumType: checksumType, + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, }) return &Response{ Data: res, diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index bd9997ee..7452eb37 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -21,7 +21,6 @@ import ( "io" "strconv" "strings" - "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -355,14 +354,20 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) } + preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource()) + resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &key, - CopySource: ©Source, - PartNumber: &partNumber, - UploadId: &uploadId, - CopySourceRange: ©SrcRange, + Bucket: &bucket, + Key: &key, + CopySource: ©Source, + PartNumber: &partNumber, + UploadId: &uploadId, + CopySourceRange: ©SrcRange, + CopySourceIfMatch: preconditionHdrs.IfMatch, + CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch, + CopySourceIfModifiedSince: preconditionHdrs.IfModSince, + CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince, }) var headers map[string]*string if err == nil && resp.CopySourceVersionId != "" { @@ -434,10 +439,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") - copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") - copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") - copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") - copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy))) taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy))) contentType := ctx.Get("Content-Type") @@ -480,35 +481,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, err } - var mtime *time.Time - if copySrcModifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcModifSince) - if err != nil { - debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err) - // TODO: check the error type for invalid values - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket) - } - mtime = &tm - } - var umtime *time.Time - if copySrcUnmodifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) - if err != nil { - debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err) - // TODO: check the error type for invalid values - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket) - } - umtime = &tm - } - metadata := utils.GetUserMetaData(&ctx.Request().Header) if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace { @@ -548,6 +520,8 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, err } + preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource()) + res, err := c.be.CopyObject(ctx.Context(), s3response.CopyObjectInput{ Bucket: &bucket, @@ -561,10 +535,10 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { Tagging: &tagging, TaggingDirective: taggingDirective, CopySource: ©Source, - CopySourceIfMatch: ©SrcIfMatch, - CopySourceIfNoneMatch: ©SrcIfNoneMatch, - CopySourceIfModifiedSince: mtime, - CopySourceIfUnmodifiedSince: umtime, + CopySourceIfMatch: preconditionHdrs.IfMatch, + CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch, + CopySourceIfModifiedSince: preconditionHdrs.IfModSince, + CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince, ExpectedBucketOwner: &acct.Access, Metadata: metadata, MetadataDirective: metaDirective, @@ -691,6 +665,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { body = bytes.NewReader([]byte{}) } + ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx) + res, err := c.be.PutObject(ctx.Context(), s3response.PutObjectInput{ Bucket: &bucket, @@ -714,6 +690,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, }) return &Response{ Headers: map[string]*string{ diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index 2a9b298d..4af140fe 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -836,42 +836,6 @@ func TestS3ApiController_CopyObject(t *testing.T) { err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), }, }, - { - name: "invalid copy source modified since", - input: testInput{ - locals: defaultLocals, - headers: map[string]string{ - "X-Amz-Copy-Source": "bucket/object", - "X-Amz-Copy-Source-If-Modified-Since": "invalid_date", - }, - }, - output: testOutput{ - response: &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: "root", - }, - }, - err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), - }, - }, - { - name: "invalid copy source unmodified since", - input: testInput{ - locals: defaultLocals, - headers: map[string]string{ - "X-Amz-Copy-Source": "bucket/object", - "X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date", - }, - }, - output: testOutput{ - response: &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: "root", - }, - }, - err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), - }, - }, { name: "invalid metadata directive", input: testInput{ diff --git a/s3api/utils/precondition.go b/s3api/utils/precondition.go new file mode 100644 index 00000000..9f3e17f4 --- /dev/null +++ b/s3api/utils/precondition.go @@ -0,0 +1,138 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "strconv" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/debuglogger" +) + +// ConditionalHeaders holds the conditional header values +type ConditionalHeaders struct { + IfMatch *string + IfNoneMatch *string + IfModSince *time.Time + IfUnmodeSince *time.Time +} + +type precondtionCfg struct { + withCopySource bool +} + +type preconditionOpt func(*precondtionCfg) + +func WithCopySource() preconditionOpt { + return func(o *precondtionCfg) { o.withCopySource = true } +} + +// ParsePreconditionHeaders parses the precondition headers: +// - If-Match +// - If-None-Match +// - If-Modified-Since +// - If-Unmodified-Since +func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders { + ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...) + ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...) + + return ConditionalHeaders{ + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, + IfModSince: ifModSince, + IfUnmodeSince: ifUnmodeSince, + } +} + +// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx +func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) { + cfg := new(precondtionCfg) + for _, opt := range opts { + opt(cfg) + } + prefix := "" + if cfg.withCopySource { + prefix = "X-Amz-Copy-Source-" + } + return GetStringPtr(ctx.Get(prefix + "If-Match")), GetStringPtr(ctx.Get(prefix + "If-None-Match")) +} + +// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since" +// headers from fiber context to *time.Time +func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) { + cfg := new(precondtionCfg) + for _, opt := range opts { + opt(cfg) + } + prefix := "" + if cfg.withCopySource { + prefix = "X-Amz-Copy-Source-" + } + + ifModSince := ctx.Get(prefix + "If-Modified-Since") + ifUnmodSince := ctx.Get(prefix + "If-Unmodified-Since") + + ifModSinceParsed := ParsePreconditionDateHeader(ifModSince) + ifUnmodSinceParsed := ParsePreconditionDateHeader(ifUnmodSince) + + return ifModSinceParsed, ifUnmodSinceParsed +} + +// ParsePreconditionDateHeader tries to parse the given date string as +// - RFC1123 +// - RFC3339 +// both are valid +func ParsePreconditionDateHeader(date string) *time.Time { + if date == "" { + return nil + } + // try to parse as RFC1123 + parsed, err := time.Parse(time.RFC1123, date) + if err == nil { + // ignore future dates + if parsed.After(time.Now()) { + return nil + } + + return &parsed + } + + // try to parse as RFC3339 + parsed, err = time.Parse(time.RFC3339, date) + if err == nil { + // ignore future dates + if parsed.After(time.Now()) { + return nil + } + + return &parsed + } + + return nil +} + +// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64 +// if parsing fails, returns nil +func ParseIfMatchSize(ctx *fiber.Ctx) *int64 { + ifMatchSizeHdr := ctx.Get("x-amz-if-match-size") + ifMatchSize, err := strconv.ParseInt(ifMatchSizeHdr, 10, 64) + if err != nil { + debuglogger.Logf("failed to parse 'x-amz-if-match-size': %s", ifMatchSizeHdr) + return nil + } + + return &ifMatchSize +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 01240a5b..5c7c2a29 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -642,59 +642,6 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type return algo, chType, nil } -// ConditionalHeaders holds the conditional header values -type ConditionalHeaders struct { - IfMatch *string - IfNoneMatch *string - IfModSince *time.Time - IfUnmodeSince *time.Time -} - -// ParsePreconditionHeaders parses the precondition headers: -// - If-Match -// - If-None-Match -// - If-Modified-Since -// - If-Unmodified-Since -func ParsePreconditionHeaders(ctx *fiber.Ctx) ConditionalHeaders { - ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx) - ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx) - - return ConditionalHeaders{ - IfMatch: ifMatch, - IfNoneMatch: ifNoneMatch, - IfModSince: ifModSince, - IfUnmodeSince: ifUnmodeSince, - } -} - -// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx -func ParsePreconditionMatchHeaders(ctx *fiber.Ctx) (*string, *string) { - return GetStringPtr(ctx.Get("If-Match")), GetStringPtr(ctx.Get("If-None-Match")) -} - -// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since" -// headers from fiber context to *time.Time -func ParsePreconditionDateHeaders(ctx *fiber.Ctx) (*time.Time, *time.Time) { - ifModSince := ctx.Get("If-Modified-Since") - ifUnmodSince := ctx.Get("If-Unmodified-Since") - - var ifModSinceParsed, ifUnmodSinceParsed *time.Time - - // the time format should be a valid RFC1123 - // if parsing fails, ignore the error and leave the value as nil - modParsed, err := time.Parse(time.RFC1123, ifModSince) - if err == nil { - ifModSinceParsed = &modParsed - } - - unmodParsed, err := time.Parse(time.RFC1123, ifUnmodSince) - if err == nil { - ifUnmodSinceParsed = &unmodParsed - } - - return ifModSinceParsed, ifUnmodSinceParsed -} - // TagLimit specifies the allowed tag count in a tag set type TagLimit int diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 4fd3481d..297972f9 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -149,6 +149,7 @@ func TestPutObject(s *S3Conf) { PutObject_with_object_lock(s) PutObject_invalid_legal_hold(s) PutObject_invalid_object_lock_mode(s) + PutObject_conditional_writes(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { PutObject_checksum_algorithm_and_header_mismatch(s) @@ -279,6 +280,7 @@ func TestDeleteObject(s *S3Conf) { DeleteObject_non_existing_dir_object(s) DeleteObject_directory_object(s) DeleteObject_non_empty_dir_obj(s) + DeleteObject_conditional_writes(s) DeleteObject_success(s) DeleteObject_success_status_code(s) } @@ -307,6 +309,7 @@ func TestCopyObject(s *S3Conf) { CopyObject_invalid_object_lock_mode(s) CopyObject_with_legal_hold(s) CopyObject_with_retention_lock(s) + CopyObject_conditional_reads(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { CopyObject_invalid_checksum_algorithm(s) @@ -398,6 +401,7 @@ func TestUploadPartCopy(s *S3Conf) { UploadPartCopy_should_copy_the_checksum(s) UploadPartCopy_should_not_copy_the_checksum(s) UploadPartCopy_should_calculate_the_checksum(s) + UploadPartCopy_conditional_reads(s) } } @@ -437,6 +441,7 @@ func TestAbortMultipartUpload(s *S3Conf) { AbortMultipartUpload_incorrect_object_key(s) AbortMultipartUpload_success(s) AbortMultipartUpload_success_status_code(s) + AbortMultipartUpload_if_match_initiated_time(s) } func TestCompleteMultipartUpload(s *S3Conf) { @@ -448,6 +453,7 @@ func TestCompleteMultipartUpload(s *S3Conf) { CompleteMultipartUpload_empty_parts(s) CompleteMultipartUpload_incorrect_parts_order(s) CompleteMultipartUpload_mpu_object_size(s) + CompleteMultipartUpload_conditional_writes(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { CompleteMultipartUpload_invalid_checksum_type(s) @@ -1053,6 +1059,7 @@ func GetIntTests() IntTests { "PutObject_with_object_lock": PutObject_with_object_lock, "PutObject_invalid_legal_hold": PutObject_invalid_legal_hold, "PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode, + "PutObject_conditional_writes": PutObject_conditional_writes, "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, @@ -1184,6 +1191,7 @@ func GetIntTests() IntTests { "DeleteObject_non_existing_object": DeleteObject_non_existing_object, "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, "DeleteObject_non_empty_dir_obj": DeleteObject_non_empty_dir_obj, + "DeleteObject_conditional_writes": DeleteObject_conditional_writes, "DeleteObject_name_too_long": DeleteObject_name_too_long, "CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object, "CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object, @@ -1211,6 +1219,7 @@ func GetIntTests() IntTests { "CopyObject_invalid_object_lock_mode": CopyObject_invalid_object_lock_mode, "CopyObject_with_legal_hold": CopyObject_with_legal_hold, "CopyObject_with_retention_lock": CopyObject_with_retention_lock, + "CopyObject_conditional_reads": CopyObject_conditional_reads, "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, "CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy, "CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum, @@ -1270,6 +1279,7 @@ func GetIntTests() IntTests { "UploadPartCopy_exceeding_copy_source_range": UploadPartCopy_exceeding_copy_source_range, "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, + "UploadPartCopy_conditional_reads": UploadPartCopy_conditional_reads, "UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum, "UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum, "UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum, @@ -1295,6 +1305,7 @@ func GetIntTests() IntTests { "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, "AbortMultipartUpload_success": AbortMultipartUpload_success, "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, + "AbortMultipartUpload_if_match_initiated_time": AbortMultipartUpload_if_match_initiated_time, "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, @@ -1302,6 +1313,7 @@ func GetIntTests() IntTests { "CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts, "CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order, "CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size, + "CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes, "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index d9552453..60e0a46c 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -3043,6 +3043,7 @@ func PutObject_invalid_legal_hold(s *S3Conf) error { return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)) }, withLock()) } + func PutObject_invalid_object_lock_mode(s *S3Conf) error { testName := "PutObject_invalid_object_lock_mode" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3057,6 +3058,78 @@ func PutObject_invalid_object_lock_mode(s *S3Conf) error { }, withLock()) } +func PutObject_conditional_writes(s *S3Conf) error { + testName := "PutObject_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + + etag := res.res.ETag + incorrectEtag := getPtr("incorrect_etag") + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + obj string + ifMatch *string + ifNoneMatch *string + err error + }{ + {obj, etag, nil, nil}, + {obj, etag, etag, errPrecond}, + {obj, etag, incorrectEtag, nil}, + {obj, incorrectEtag, incorrectEtag, errPrecond}, + {obj, incorrectEtag, etag, errPrecond}, + {obj, incorrectEtag, nil, errPrecond}, + {obj, nil, incorrectEtag, nil}, + {obj, nil, etag, errPrecond}, + {obj, nil, nil, nil}, + // should ignore the precondition headers if + // an object with the given name doesn't exist + {"obj-1", incorrectEtag, etag, nil}, + {"obj-2", etag, etag, nil}, + {"obj-3", etag, incorrectEtag, nil}, + {"obj-4", incorrectEtag, nil, nil}, + {"obj-5", nil, etag, nil}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &test.obj, + Body: bytes.NewReader([]byte("dummy")), + IfMatch: test.ifMatch, + IfNoneMatch: test.ifNoneMatch, + }, s3client) + if err == nil { + // azure blob storage generates different ETags for + // the exact same data. + // to avoid ETag collision reassign the etag value + *etag = *res.res.ETag + } + if test.err == nil && err != nil { + return fmt.Errorf("test case %v: expected no error, instead got %w", i, err) + } + if test.err != nil { + apierr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("test case %v: invalid error type: %w", i, test.err) + } + + if err := checkApiErr(err, apierr); err != nil { + return fmt.Errorf("test case %v: %w", i, err) + } + } + } + + return nil + }) +} + func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { testName := "PutObject_checksum_algorithm_and_header_mismatch" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6895,6 +6968,98 @@ func DeleteObject_non_empty_dir_obj(s *S3Conf) error { }) } +func DeleteObject_conditional_writes(s *S3Conf) error { + testName := "DeleteObject_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + var etag *string = getPtr("") + var size *int64 = getPtr(int64(0)) + var modTime *time.Time = getPtr(time.Now()) + + createObj := func() error { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + + // get the exact LastModified time + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + *etag = *res.res.ETag + *size = *res.res.Size + *modTime = *out.LastModified + + return nil + } + + err := createObj() + if err != nil { + return err + } + + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + ifMatch *string + size *int64 + modTime *time.Time + err error + }{ + // no error cases + {etag, size, modTime, nil}, + {etag, nil, nil, nil}, + {nil, size, nil, nil}, + {nil, nil, modTime, nil}, + {etag, size, nil, nil}, + {etag, nil, modTime, nil}, + {nil, size, modTime, nil}, + // error cases + {getPtr("incorrect_etag"), nil, nil, errPrecond}, + {nil, getPtr(int64(23234)), nil, errPrecond}, + {nil, nil, getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond}, + {getPtr("incorrect_etag"), getPtr(int64(23234)), nil, errPrecond}, + {getPtr("incorrect_etag"), getPtr(int64(23234)), getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond}, + } { + err := createObj() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + IfMatch: test.ifMatch, + IfMatchSize: test.size, + IfMatchLastModifiedTime: test.modTime, + }) + cancel() + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func DeleteObject_directory_not_empty(s *S3Conf) error { testName := "DeleteObject_directory_not_empty" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -8022,6 +8187,156 @@ func CopyObject_with_retention_lock(s *S3Conf) error { }, withLock()) } +func CopyObject_conditional_reads(s *S3Conf) error { + testName := "CopyObject_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := s3err.GetAPIError(s3err.ErrNotModified) + errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err error + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: getPtr("dst-obj"), + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)), + CopySourceIfMatch: test.ifmatch, + CopySourceIfNoneMatch: test.ifnonematch, + CopySourceIfModifiedSince: test.ifmodifiedsince, + CopySourceIfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func CopyObject_invalid_checksum_algorithm(s *S3Conf) error { testName := "CopyObject_invalid_checksum_algorithm" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -10476,6 +10791,164 @@ func UploadPartCopy_by_range_success(s *S3Conf) error { }) } +func UploadPartCopy_conditional_reads(s *S3Conf) error { + testName := "UploadPartCopy_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := s3err.GetAPIError(s3err.ErrNotModified) + errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err error + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + } { + mpKey := "mp-key" + mp, err := createMp(s3client, bucket, mpKey) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &mpKey, + UploadId: mp.UploadId, + PartNumber: getPtr(int32(1)), + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)), + CopySourceIfMatch: test.ifmatch, + CopySourceIfNoneMatch: test.ifnonematch, + CopySourceIfModifiedSince: test.ifmodifiedsince, + CopySourceIfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error { testName := "UploadPartCopy_should_copy_the_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -11427,6 +11900,87 @@ func AbortMultipartUpload_success_status_code(s *S3Conf) error { }) } +func AbortMultipartUpload_if_match_initiated_time(s *S3Conf) error { + testName := "AbortMultipartUpload_if_match_initiated_time" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + var initiated *time.Time = getPtr(time.Now()) + + // createMpUpload creates a multipart uplod + // and retruns the uploadId and creation date + abortMp := func(date *time.Time) error { + mpObj := "my-obj" + mp, err := createMp(s3client, bucket, mpObj) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + var initiatedTime *time.Time + + for _, up := range res.Uploads { + if getString(up.UploadId) == getString(mp.UploadId) { + initiatedTime = up.Initiated + break + } + } + + if initiatedTime == nil { + return fmt.Errorf("unexpected err: the multipart upload is not found") + } + + *initiated = *initiatedTime + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &mpObj, + UploadId: mp.UploadId, + IfMatchInitiatedTime: date, + }) + cancel() + + return err + } + + for i, test := range []struct { + date *time.Time + err error + }{ + {nil, nil}, + // match: success case + {initiated, nil}, + // should ignore future dates + {getPtr(initiated.AddDate(1, 0, 0)), nil}, + // should fail if the initation date doesn't match + {getPtr(initiated.AddDate(-1, 0, 1)), s3err.GetAPIError(s3err.ErrPreconditionFailed)}, + } { + err := abortMp(test.date) + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error { testName := "CompletedMultipartUpload_non_existing_bucket" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -12593,6 +13147,100 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error { }) } +func CompleteMultipartUpload_conditional_writes(s *S3Conf) error { + testName := "CompleteMultipartUpload_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + etag := getPtr("") + incorrectEtag := getPtr("incorrect_etag") + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + obj string + ifMatch *string + ifNoneMatch *string + err error + }{ + {obj, etag, nil, nil}, + {obj, etag, etag, errPrecond}, + {obj, etag, incorrectEtag, nil}, + {obj, incorrectEtag, incorrectEtag, errPrecond}, + {obj, incorrectEtag, etag, errPrecond}, + {obj, incorrectEtag, nil, errPrecond}, + {obj, nil, incorrectEtag, nil}, + {obj, nil, etag, errPrecond}, + {obj, nil, nil, nil}, + // should ignore the precondition headers if + // an object with the given name doesn't exist + {"obj-1", incorrectEtag, etag, nil}, + {"obj-2", etag, etag, nil}, + {"obj-3", etag, incorrectEtag, nil}, + {"obj-4", incorrectEtag, nil, nil}, + {"obj-5", nil, etag, nil}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + // azure blob storage generates different ETags for + // the exact same data. + // to avoid ETag collision reassign the etag value + *etag = *res.res.ETag + + mp, err := createMp(s3client, bucket, test.obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, test.obj, *mp.UploadId) + if err != nil { + return err + } + + part := parts[0] + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &test.obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: part.ETag, + PartNumber: getPtr(int32(1)), + ChecksumCRC64NVME: part.ChecksumCRC64NVME, + }, + }, + }, + IfMatch: test.ifMatch, + IfNoneMatch: test.ifNoneMatch, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %v: expected no error, instead got %w", i, err) + } + if test.err != nil { + apierr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("test case %v: invalid error type: %w", i, test.err) + } + + if err := checkApiErr(err, apierr); err != nil { + return fmt.Errorf("test case %v: %w", i, err) + } + } + } + + return nil + }) +} + func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { testName := "CompleteMultipartUpload_invalid_part_number" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index b57311e9..7ad0677c 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -518,11 +518,15 @@ type putObjectOutput struct { } func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) { - data := make([]byte, lgth) - rand.Read(data) - csum := sha256.Sum256(data) - r := bytes.NewReader(data) - input.Body = r + var csum [32]byte + var data []byte + if input.Body == nil { + data = make([]byte, lgth) + rand.Read(data) + csum = sha256.Sum256(data) + r := bytes.NewReader(data) + input.Body = r + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := client.PutObject(ctx, input)