diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 0c3286d2..5c2ec8d6 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -301,6 +301,11 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s return s3response.PutObjectOutput{}, err } + err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + metadata := parseMetadata(po.Metadata) // Store the "Expires" property in the object metadata @@ -850,6 +855,42 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input } func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil { + // evaluate the preconditions before deleting the object + props, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: input.Bucket, + Key: input.Key, + }) + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + // if object doesn't exist, skip preconditions + // if unexpected error shows up, return the error + return nil, err + } + if err == nil { + var etag string + if props.ETag != nil { + etag = *props.ETag + } + var lastMod time.Time + if props.LastModified != nil { + lastMod = *props.LastModified + } + var size int64 + if props.ContentLength != nil { + size = *props.ContentLength + } + err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size, + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + if err != nil { + return nil, err + } + } + } + _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) if err != nil { azerr, ok := err.(*azcore.ResponseError) @@ -899,6 +940,26 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu if err != nil { return s3response.CopyObjectOutput{}, err } + + srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + + if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) { + _, err = az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModifiedSince: input.CopySourceIfModifiedSince, + IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + } + if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource { if input.MetadataDirective != types.MetadataDirectiveReplace { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest) @@ -977,11 +1038,6 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu }, nil } - srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource) - if err != nil { - return s3response.CopyObjectOutput{}, err - } - // Get the source object downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil) if err != nil { @@ -1210,7 +1266,7 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3 func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) { client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) if err != nil { - return s3response.CopyPartResult{}, nil + return s3response.CopyPartResult{}, err } if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil { @@ -1413,6 +1469,22 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa // Cleans up the initiated multipart upload in .sgwtmp namespace func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) + + if input.IfMatchInitiatedTime != nil { + client, err := az.getBlobClient(*input.Bucket, tmpPath) + if err != nil { + return err + } + + resp, err := client.GetProperties(ctx, nil) + if err != nil { + return azureErrToS3Err(err) + } + + if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } _, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil) if err != nil { return parseMpError(err) @@ -1440,6 +1512,11 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { var res s3response.CompleteMultipartUploadResult + err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", err + } + tmpPath := createMetaTmpPath(*input.Key, *input.UploadId) blobClient, err := az.getBlobClient(*input.Bucket, tmpPath) if err != nil { @@ -2058,6 +2135,29 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string return nil } +func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error { + if areNils(ifMatch, ifNoneMatch) { + return nil + } + // call HeadObject to evaluate preconditions + // if object doesn't exist, move forward with the object creation + // otherwise return the error + _, err := az.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: bucket, + Key: object, + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, + }) + if errors.Is(err, s3err.GetAPIError(s3err.ErrNotModified)) { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) { + return err + } + + return nil +} + func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) { data, ok := meta[string(key)] if !ok { @@ -2105,3 +2205,13 @@ func convertAzureEtag(etag *azcore.ETag) string { return *backend.TrimEtag(str) + "-1" } + +func areNils[T any](args ...*T) bool { + for _, arg := range args { + if arg != nil { + return false + } + } + + return true +} diff --git a/backend/common.go b/backend/common.go index fd3a68ce..f687c1e0 100644 --- a/backend/common.go +++ b/backend/common.go @@ -532,3 +532,41 @@ func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreCond return nil } + +// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions +func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error { + if ifMatch != nil && *ifMatch != etag { + return errPreconditionFailed + } + if ifNoneMatch != nil && *ifNoneMatch == etag { + return errPreconditionFailed + } + + return nil +} + +type ObjectDeletePreconditions struct { + IfMatch *string + IfMatchLastModTime *time.Time + IfMatchSize *int64 +} + +// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject +func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error { + ifMatch := preconditions.IfMatch + if ifMatch != nil && *ifMatch != etag { + return errPreconditionFailed + } + + ifMatchTime := preconditions.IfMatchLastModTime + if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() { + return errPreconditionFailed + } + + ifMatchSize := preconditions.IfMatchSize + if ifMatchSize != nil && *ifMatchSize != size { + return errPreconditionFailed + } + + return nil +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 91b9a9d6..0a4b9495 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1401,6 +1401,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return res, "", err } + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + if err == nil { + err = backend.EvaluateMatchPreconditions(string(b), input.IfMatch, input.IfNoneMatch) + if err != nil { + return res, "", err + } + } + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) @@ -2009,11 +2017,17 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp sum := sha256.Sum256([]byte(object)) objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) - _, err = os.Stat(filepath.Join(objdir, uploadID)) + f, err := os.Stat(filepath.Join(objdir, uploadID)) if err != nil { return s3err.GetAPIError(s3err.ErrNoSuchUpload) } + if mpu.IfMatchInitiatedTime != nil { + if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } + err = os.RemoveAll(filepath.Join(objdir, uploadID)) if err != nil { return fmt.Errorf("remove multipart upload container: %w", err) @@ -2600,6 +2614,32 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) return s3response.CopyPartResult{}, err } + srcf, err := os.Open(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if err != nil { + return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) + } + defer srcf.Close() + + // evaluate preconditions + b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil { + srcEtag = "" + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: upi.CopySourceIfMatch, + IfNoneMatch: upi.CopySourceIfNoneMatch, + IfModSince: upi.CopySourceIfModifiedSince, + IfUnmodeSince: upi.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyPartResult{}, err + } + f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir), *upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile) if err != nil { @@ -2610,15 +2650,6 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } defer f.cleanup() - srcf, err := os.Open(objPath) - if errors.Is(err, fs.ErrNotExist) { - return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) - } - if err != nil { - return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err) - } - defer srcf.Close() - rdr := io.NewSectionReader(srcf, startOffset, length) hash := md5.New() tr := io.TeeReader(rdr, hash) @@ -2748,6 +2779,15 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3 name := filepath.Join(*po.Bucket, *po.Key) + // evaluate preconditions + etagBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, etagkey) + if err == nil { + err := backend.EvaluateMatchPreconditions(string(etagBytes), po.IfMatch, po.IfNoneMatch) + if err != nil { + return s3response.PutObjectOutput{}, err + } + } + uid, gid, doChown := p.getChownIDs(acct) contentLength := int64(0) @@ -3073,6 +3113,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, err } + evalPreconditions := func(f os.FileInfo, bucket, object string) error { + var err error + if f == nil { + f, err = os.Stat(filepath.Join(bucket, object)) + if err != nil { + return nil + } + } + + b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) + etag := string(b) + if err != nil { + etag = "" + } + + // evaluate preconditions + return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(), + backend.ObjectDeletePreconditions{ + IfMatch: input.IfMatch, + IfMatchLastModTime: input.IfMatchLastModifiedTime, + IfMatchSize: input.IfMatchSize, + }) + } + // Directory objects can't have versions if !isDir && p.versioningEnabled() && vStatus != "" { if getString(input.VersionId) == "" { @@ -3089,6 +3153,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} @@ -3148,6 +3217,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } if string(vId) == *input.VersionId { + // evaluate preconditions + err := evalPreconditions(nil, bucket, object) + if err != nil { + return nil, err + } // if the specified VersionId is the same as in the latest version, // remove the latest version, find the latest version from the versioning // directory and move to the place of the deleted object, to make it the latest @@ -3242,6 +3316,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( }, nil } + err = evalPreconditions(nil, versionPath, *input.VersionId) + if err != nil { + return nil, err + } + isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId) err = os.Remove(filepath.Join(versionPath, *input.VersionId)) @@ -3289,6 +3368,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } + err = evalPreconditions(fi, bucket, object) + if err != nil { + return nil, err + } + err = os.Remove(objpath) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) @@ -4031,6 +4115,22 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } + b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey) + srcEtag := string(b) + if err != nil { + srcEtag = "" + } + + err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ + IfMatch: input.CopySourceIfMatch, + IfNoneMatch: input.CopySourceIfNoneMatch, + IfModSince: input.CopySourceIfModifiedSince, + IfUnmodeSince: input.CopySourceIfUnmodifiedSince, + }) + if err != nil { + return s3response.CopyObjectOutput{}, err + } + mdmap := make(map[string]string) p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap) diff --git a/s3api/controllers/object-delete.go b/s3api/controllers/object-delete.go index d625fe67..bfb5367d 100644 --- a/s3api/controllers/object-delete.go +++ b/s3api/controllers/object-delete.go @@ -69,6 +69,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) uploadId := ctx.Query("uploadId") + ifMatchInitiatedTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Initiated-Time")) acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx) @@ -96,9 +97,10 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error) err = c.be.AbortMultipartUpload(ctx.Context(), &s3.AbortMultipartUploadInput{ - UploadId: &uploadId, - Bucket: &bucket, - Key: &key, + UploadId: &uploadId, + Bucket: &bucket, + Key: &key, + IfMatchInitiatedTime: ifMatchInitiatedTime, }) return &Response{ MetaOpts: &MetaOptions{ @@ -113,6 +115,9 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) versionId := ctx.Query("versionId") bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true") + ifMatch := utils.GetStringPtr(ctx.Get("If-Match")) + ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time")) + ifMatchSize := utils.ParseIfMatchSize(ctx) // context locals acct := utils.ContextKeyAccount.Get(ctx).(auth.Account) isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool) @@ -165,9 +170,12 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) { res, err := c.be.DeleteObject(ctx.Context(), &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &key, - VersionId: &versionId, + Bucket: &bucket, + Key: &key, + VersionId: &versionId, + IfMatch: ifMatch, + IfMatchLastModifiedTime: ifMatchLastModTime, + IfMatchSize: ifMatchSize, }) if err != nil { return &Response{ diff --git a/s3api/controllers/object-post.go b/s3api/controllers/object-post.go index 35614efd..4f4df452 100644 --- a/s3api/controllers/object-post.go +++ b/s3api/controllers/object-post.go @@ -323,6 +323,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err }, err } + ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx) + res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{ Bucket: &bucket, @@ -338,6 +340,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), ChecksumType: checksumType, + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, }) return &Response{ Data: res, diff --git a/s3api/controllers/object-put.go b/s3api/controllers/object-put.go index bd9997ee..7452eb37 100644 --- a/s3api/controllers/object-put.go +++ b/s3api/controllers/object-put.go @@ -21,7 +21,6 @@ import ( "io" "strconv" "strings" - "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -355,14 +354,20 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) { }, s3err.GetAPIError(s3err.ErrInvalidPartNumber) } + preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource()) + resp, err := c.be.UploadPartCopy(ctx.Context(), &s3.UploadPartCopyInput{ - Bucket: &bucket, - Key: &key, - CopySource: ©Source, - PartNumber: &partNumber, - UploadId: &uploadId, - CopySourceRange: ©SrcRange, + Bucket: &bucket, + Key: &key, + CopySource: ©Source, + PartNumber: &partNumber, + UploadId: &uploadId, + CopySourceRange: ©SrcRange, + CopySourceIfMatch: preconditionHdrs.IfMatch, + CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch, + CopySourceIfModifiedSince: preconditionHdrs.IfModSince, + CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince, }) var headers map[string]*string if err == nil && resp.CopySourceVersionId != "" { @@ -434,10 +439,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { bucket := ctx.Params("bucket") key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket)) copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/") - copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match") - copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match") - copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since") - copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since") metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy))) taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy))) contentType := ctx.Get("Content-Type") @@ -480,35 +481,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, err } - var mtime *time.Time - if copySrcModifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcModifSince) - if err != nil { - debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err) - // TODO: check the error type for invalid values - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket) - } - mtime = &tm - } - var umtime *time.Time - if copySrcUnmodifSince != "" { - tm, err := time.Parse(iso8601Format, copySrcUnmodifSince) - if err != nil { - debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err) - // TODO: check the error type for invalid values - return &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: parsedAcl.Owner, - }, - }, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket) - } - umtime = &tm - } - metadata := utils.GetUserMetaData(&ctx.Request().Header) if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace { @@ -548,6 +520,8 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { }, err } + preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource()) + res, err := c.be.CopyObject(ctx.Context(), s3response.CopyObjectInput{ Bucket: &bucket, @@ -561,10 +535,10 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) { Tagging: &tagging, TaggingDirective: taggingDirective, CopySource: ©Source, - CopySourceIfMatch: ©SrcIfMatch, - CopySourceIfNoneMatch: ©SrcIfNoneMatch, - CopySourceIfModifiedSince: mtime, - CopySourceIfUnmodifiedSince: umtime, + CopySourceIfMatch: preconditionHdrs.IfMatch, + CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch, + CopySourceIfModifiedSince: preconditionHdrs.IfModSince, + CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince, ExpectedBucketOwner: &acct.Access, Metadata: metadata, MetadataDirective: metaDirective, @@ -691,6 +665,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { body = bytes.NewReader([]byte{}) } + ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx) + res, err := c.be.PutObject(ctx.Context(), s3response.PutObjectInput{ Bucket: &bucket, @@ -714,6 +690,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) { ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]), ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]), + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, }) return &Response{ Headers: map[string]*string{ diff --git a/s3api/controllers/object-put_test.go b/s3api/controllers/object-put_test.go index 2a9b298d..4af140fe 100644 --- a/s3api/controllers/object-put_test.go +++ b/s3api/controllers/object-put_test.go @@ -836,42 +836,6 @@ func TestS3ApiController_CopyObject(t *testing.T) { err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), }, }, - { - name: "invalid copy source modified since", - input: testInput{ - locals: defaultLocals, - headers: map[string]string{ - "X-Amz-Copy-Source": "bucket/object", - "X-Amz-Copy-Source-If-Modified-Since": "invalid_date", - }, - }, - output: testOutput{ - response: &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: "root", - }, - }, - err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), - }, - }, - { - name: "invalid copy source unmodified since", - input: testInput{ - locals: defaultLocals, - headers: map[string]string{ - "X-Amz-Copy-Source": "bucket/object", - "X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date", - }, - }, - output: testOutput{ - response: &Response{ - MetaOpts: &MetaOptions{ - BucketOwner: "root", - }, - }, - err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket), - }, - }, { name: "invalid metadata directive", input: testInput{ diff --git a/s3api/utils/precondition.go b/s3api/utils/precondition.go new file mode 100644 index 00000000..9f3e17f4 --- /dev/null +++ b/s3api/utils/precondition.go @@ -0,0 +1,138 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "strconv" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/versity/versitygw/debuglogger" +) + +// ConditionalHeaders holds the conditional header values +type ConditionalHeaders struct { + IfMatch *string + IfNoneMatch *string + IfModSince *time.Time + IfUnmodeSince *time.Time +} + +type precondtionCfg struct { + withCopySource bool +} + +type preconditionOpt func(*precondtionCfg) + +func WithCopySource() preconditionOpt { + return func(o *precondtionCfg) { o.withCopySource = true } +} + +// ParsePreconditionHeaders parses the precondition headers: +// - If-Match +// - If-None-Match +// - If-Modified-Since +// - If-Unmodified-Since +func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders { + ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...) + ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...) + + return ConditionalHeaders{ + IfMatch: ifMatch, + IfNoneMatch: ifNoneMatch, + IfModSince: ifModSince, + IfUnmodeSince: ifUnmodeSince, + } +} + +// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx +func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) { + cfg := new(precondtionCfg) + for _, opt := range opts { + opt(cfg) + } + prefix := "" + if cfg.withCopySource { + prefix = "X-Amz-Copy-Source-" + } + return GetStringPtr(ctx.Get(prefix + "If-Match")), GetStringPtr(ctx.Get(prefix + "If-None-Match")) +} + +// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since" +// headers from fiber context to *time.Time +func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) { + cfg := new(precondtionCfg) + for _, opt := range opts { + opt(cfg) + } + prefix := "" + if cfg.withCopySource { + prefix = "X-Amz-Copy-Source-" + } + + ifModSince := ctx.Get(prefix + "If-Modified-Since") + ifUnmodSince := ctx.Get(prefix + "If-Unmodified-Since") + + ifModSinceParsed := ParsePreconditionDateHeader(ifModSince) + ifUnmodSinceParsed := ParsePreconditionDateHeader(ifUnmodSince) + + return ifModSinceParsed, ifUnmodSinceParsed +} + +// ParsePreconditionDateHeader tries to parse the given date string as +// - RFC1123 +// - RFC3339 +// both are valid +func ParsePreconditionDateHeader(date string) *time.Time { + if date == "" { + return nil + } + // try to parse as RFC1123 + parsed, err := time.Parse(time.RFC1123, date) + if err == nil { + // ignore future dates + if parsed.After(time.Now()) { + return nil + } + + return &parsed + } + + // try to parse as RFC3339 + parsed, err = time.Parse(time.RFC3339, date) + if err == nil { + // ignore future dates + if parsed.After(time.Now()) { + return nil + } + + return &parsed + } + + return nil +} + +// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64 +// if parsing fails, returns nil +func ParseIfMatchSize(ctx *fiber.Ctx) *int64 { + ifMatchSizeHdr := ctx.Get("x-amz-if-match-size") + ifMatchSize, err := strconv.ParseInt(ifMatchSizeHdr, 10, 64) + if err != nil { + debuglogger.Logf("failed to parse 'x-amz-if-match-size': %s", ifMatchSizeHdr) + return nil + } + + return &ifMatchSize +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 01240a5b..5c7c2a29 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -642,59 +642,6 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type return algo, chType, nil } -// ConditionalHeaders holds the conditional header values -type ConditionalHeaders struct { - IfMatch *string - IfNoneMatch *string - IfModSince *time.Time - IfUnmodeSince *time.Time -} - -// ParsePreconditionHeaders parses the precondition headers: -// - If-Match -// - If-None-Match -// - If-Modified-Since -// - If-Unmodified-Since -func ParsePreconditionHeaders(ctx *fiber.Ctx) ConditionalHeaders { - ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx) - ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx) - - return ConditionalHeaders{ - IfMatch: ifMatch, - IfNoneMatch: ifNoneMatch, - IfModSince: ifModSince, - IfUnmodeSince: ifUnmodeSince, - } -} - -// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx -func ParsePreconditionMatchHeaders(ctx *fiber.Ctx) (*string, *string) { - return GetStringPtr(ctx.Get("If-Match")), GetStringPtr(ctx.Get("If-None-Match")) -} - -// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since" -// headers from fiber context to *time.Time -func ParsePreconditionDateHeaders(ctx *fiber.Ctx) (*time.Time, *time.Time) { - ifModSince := ctx.Get("If-Modified-Since") - ifUnmodSince := ctx.Get("If-Unmodified-Since") - - var ifModSinceParsed, ifUnmodSinceParsed *time.Time - - // the time format should be a valid RFC1123 - // if parsing fails, ignore the error and leave the value as nil - modParsed, err := time.Parse(time.RFC1123, ifModSince) - if err == nil { - ifModSinceParsed = &modParsed - } - - unmodParsed, err := time.Parse(time.RFC1123, ifUnmodSince) - if err == nil { - ifUnmodSinceParsed = &unmodParsed - } - - return ifModSinceParsed, ifUnmodSinceParsed -} - // TagLimit specifies the allowed tag count in a tag set type TagLimit int diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 4fd3481d..297972f9 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -149,6 +149,7 @@ func TestPutObject(s *S3Conf) { PutObject_with_object_lock(s) PutObject_invalid_legal_hold(s) PutObject_invalid_object_lock_mode(s) + PutObject_conditional_writes(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { PutObject_checksum_algorithm_and_header_mismatch(s) @@ -279,6 +280,7 @@ func TestDeleteObject(s *S3Conf) { DeleteObject_non_existing_dir_object(s) DeleteObject_directory_object(s) DeleteObject_non_empty_dir_obj(s) + DeleteObject_conditional_writes(s) DeleteObject_success(s) DeleteObject_success_status_code(s) } @@ -307,6 +309,7 @@ func TestCopyObject(s *S3Conf) { CopyObject_invalid_object_lock_mode(s) CopyObject_with_legal_hold(s) CopyObject_with_retention_lock(s) + CopyObject_conditional_reads(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { CopyObject_invalid_checksum_algorithm(s) @@ -398,6 +401,7 @@ func TestUploadPartCopy(s *S3Conf) { UploadPartCopy_should_copy_the_checksum(s) UploadPartCopy_should_not_copy_the_checksum(s) UploadPartCopy_should_calculate_the_checksum(s) + UploadPartCopy_conditional_reads(s) } } @@ -437,6 +441,7 @@ func TestAbortMultipartUpload(s *S3Conf) { AbortMultipartUpload_incorrect_object_key(s) AbortMultipartUpload_success(s) AbortMultipartUpload_success_status_code(s) + AbortMultipartUpload_if_match_initiated_time(s) } func TestCompleteMultipartUpload(s *S3Conf) { @@ -448,6 +453,7 @@ func TestCompleteMultipartUpload(s *S3Conf) { CompleteMultipartUpload_empty_parts(s) CompleteMultipartUpload_incorrect_parts_order(s) CompleteMultipartUpload_mpu_object_size(s) + CompleteMultipartUpload_conditional_writes(s) //TODO: remove the condition after implementing checksums in azure if !s.azureTests { CompleteMultipartUpload_invalid_checksum_type(s) @@ -1053,6 +1059,7 @@ func GetIntTests() IntTests { "PutObject_with_object_lock": PutObject_with_object_lock, "PutObject_invalid_legal_hold": PutObject_invalid_legal_hold, "PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode, + "PutObject_conditional_writes": PutObject_conditional_writes, "PutObject_invalid_credentials": PutObject_invalid_credentials, "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, @@ -1184,6 +1191,7 @@ func GetIntTests() IntTests { "DeleteObject_non_existing_object": DeleteObject_non_existing_object, "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, "DeleteObject_non_empty_dir_obj": DeleteObject_non_empty_dir_obj, + "DeleteObject_conditional_writes": DeleteObject_conditional_writes, "DeleteObject_name_too_long": DeleteObject_name_too_long, "CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object, "CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object, @@ -1211,6 +1219,7 @@ func GetIntTests() IntTests { "CopyObject_invalid_object_lock_mode": CopyObject_invalid_object_lock_mode, "CopyObject_with_legal_hold": CopyObject_with_legal_hold, "CopyObject_with_retention_lock": CopyObject_with_retention_lock, + "CopyObject_conditional_reads": CopyObject_conditional_reads, "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, "CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy, "CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum, @@ -1270,6 +1279,7 @@ func GetIntTests() IntTests { "UploadPartCopy_exceeding_copy_source_range": UploadPartCopy_exceeding_copy_source_range, "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, + "UploadPartCopy_conditional_reads": UploadPartCopy_conditional_reads, "UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum, "UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum, "UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum, @@ -1295,6 +1305,7 @@ func GetIntTests() IntTests { "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, "AbortMultipartUpload_success": AbortMultipartUpload_success, "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, + "AbortMultipartUpload_if_match_initiated_time": AbortMultipartUpload_if_match_initiated_time, "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, @@ -1302,6 +1313,7 @@ func GetIntTests() IntTests { "CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts, "CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order, "CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size, + "CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes, "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index d9552453..60e0a46c 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -3043,6 +3043,7 @@ func PutObject_invalid_legal_hold(s *S3Conf) error { return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus)) }, withLock()) } + func PutObject_invalid_object_lock_mode(s *S3Conf) error { testName := "PutObject_invalid_object_lock_mode" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3057,6 +3058,78 @@ func PutObject_invalid_object_lock_mode(s *S3Conf) error { }, withLock()) } +func PutObject_conditional_writes(s *S3Conf) error { + testName := "PutObject_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + + etag := res.res.ETag + incorrectEtag := getPtr("incorrect_etag") + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + obj string + ifMatch *string + ifNoneMatch *string + err error + }{ + {obj, etag, nil, nil}, + {obj, etag, etag, errPrecond}, + {obj, etag, incorrectEtag, nil}, + {obj, incorrectEtag, incorrectEtag, errPrecond}, + {obj, incorrectEtag, etag, errPrecond}, + {obj, incorrectEtag, nil, errPrecond}, + {obj, nil, incorrectEtag, nil}, + {obj, nil, etag, errPrecond}, + {obj, nil, nil, nil}, + // should ignore the precondition headers if + // an object with the given name doesn't exist + {"obj-1", incorrectEtag, etag, nil}, + {"obj-2", etag, etag, nil}, + {"obj-3", etag, incorrectEtag, nil}, + {"obj-4", incorrectEtag, nil, nil}, + {"obj-5", nil, etag, nil}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &test.obj, + Body: bytes.NewReader([]byte("dummy")), + IfMatch: test.ifMatch, + IfNoneMatch: test.ifNoneMatch, + }, s3client) + if err == nil { + // azure blob storage generates different ETags for + // the exact same data. + // to avoid ETag collision reassign the etag value + *etag = *res.res.ETag + } + if test.err == nil && err != nil { + return fmt.Errorf("test case %v: expected no error, instead got %w", i, err) + } + if test.err != nil { + apierr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("test case %v: invalid error type: %w", i, test.err) + } + + if err := checkApiErr(err, apierr); err != nil { + return fmt.Errorf("test case %v: %w", i, err) + } + } + } + + return nil + }) +} + func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error { testName := "PutObject_checksum_algorithm_and_header_mismatch" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -6895,6 +6968,98 @@ func DeleteObject_non_empty_dir_obj(s *S3Conf) error { }) } +func DeleteObject_conditional_writes(s *S3Conf) error { + testName := "DeleteObject_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + var etag *string = getPtr("") + var size *int64 = getPtr(int64(0)) + var modTime *time.Time = getPtr(time.Now()) + + createObj := func() error { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + + // get the exact LastModified time + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + *etag = *res.res.ETag + *size = *res.res.Size + *modTime = *out.LastModified + + return nil + } + + err := createObj() + if err != nil { + return err + } + + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + ifMatch *string + size *int64 + modTime *time.Time + err error + }{ + // no error cases + {etag, size, modTime, nil}, + {etag, nil, nil, nil}, + {nil, size, nil, nil}, + {nil, nil, modTime, nil}, + {etag, size, nil, nil}, + {etag, nil, modTime, nil}, + {nil, size, modTime, nil}, + // error cases + {getPtr("incorrect_etag"), nil, nil, errPrecond}, + {nil, getPtr(int64(23234)), nil, errPrecond}, + {nil, nil, getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond}, + {getPtr("incorrect_etag"), getPtr(int64(23234)), nil, errPrecond}, + {getPtr("incorrect_etag"), getPtr(int64(23234)), getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond}, + } { + err := createObj() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + IfMatch: test.ifMatch, + IfMatchSize: test.size, + IfMatchLastModifiedTime: test.modTime, + }) + cancel() + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func DeleteObject_directory_not_empty(s *S3Conf) error { testName := "DeleteObject_directory_not_empty" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -8022,6 +8187,156 @@ func CopyObject_with_retention_lock(s *S3Conf) error { }, withLock()) } +func CopyObject_conditional_reads(s *S3Conf) error { + testName := "CopyObject_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := s3err.GetAPIError(s3err.ErrNotModified) + errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err error + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + } { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: getPtr("dst-obj"), + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)), + CopySourceIfMatch: test.ifmatch, + CopySourceIfNoneMatch: test.ifnonematch, + CopySourceIfModifiedSince: test.ifmodifiedsince, + CopySourceIfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func CopyObject_invalid_checksum_algorithm(s *S3Conf) error { testName := "CopyObject_invalid_checksum_algorithm" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -10476,6 +10791,164 @@ func UploadPartCopy_by_range_success(s *S3Conf) error { }) } +func UploadPartCopy_conditional_reads(s *S3Conf) error { + testName := "UploadPartCopy_conditional_reads" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + key := "my-obj" + obj, err := putObjectWithData(10, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + }, s3client) + if err != nil { + return err + } + + errMod := s3err.GetAPIError(s3err.ErrNotModified) + errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + // sleep one second to get dates before and after + // the object creation + time.Sleep(time.Second * 1) + + before := time.Now().AddDate(0, 0, -3) + after := time.Now() + etag := obj.res.ETag + + for i, test := range []struct { + ifmatch *string + ifnonematch *string + ifmodifiedsince *time.Time + ifunmodifiedsince *time.Time + err error + }{ + // all the cases when preconditions are either empty, true or false + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond}, + {getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond}, + + {getPtr("invalid_etag"), etag, &before, &before, errCond}, + {getPtr("invalid_etag"), etag, &before, &after, errCond}, + {getPtr("invalid_etag"), etag, &before, nil, errCond}, + {getPtr("invalid_etag"), etag, &after, &before, errCond}, + {getPtr("invalid_etag"), etag, &after, &after, errCond}, + {getPtr("invalid_etag"), etag, &after, nil, errCond}, + {getPtr("invalid_etag"), etag, nil, &before, errCond}, + {getPtr("invalid_etag"), etag, nil, &after, errCond}, + {getPtr("invalid_etag"), etag, nil, nil, errCond}, + + {getPtr("invalid_etag"), nil, &before, &before, errCond}, + {getPtr("invalid_etag"), nil, &before, &after, errCond}, + {getPtr("invalid_etag"), nil, &before, nil, errCond}, + {getPtr("invalid_etag"), nil, &after, &before, errCond}, + {getPtr("invalid_etag"), nil, &after, &after, errCond}, + {getPtr("invalid_etag"), nil, &after, nil, errCond}, + {getPtr("invalid_etag"), nil, nil, &before, errCond}, + {getPtr("invalid_etag"), nil, nil, &after, errCond}, + {getPtr("invalid_etag"), nil, nil, nil, errCond}, + + {etag, getPtr("invalid_etag"), &before, &before, nil}, + {etag, getPtr("invalid_etag"), &before, &after, nil}, + {etag, getPtr("invalid_etag"), &before, nil, nil}, + {etag, getPtr("invalid_etag"), &after, &before, nil}, + {etag, getPtr("invalid_etag"), &after, &after, nil}, + {etag, getPtr("invalid_etag"), &after, nil, nil}, + {etag, getPtr("invalid_etag"), nil, &before, nil}, + {etag, getPtr("invalid_etag"), nil, &after, nil}, + {etag, getPtr("invalid_etag"), nil, nil, nil}, + + {etag, etag, &before, &before, errMod}, + {etag, etag, &before, &after, errMod}, + {etag, etag, &before, nil, errMod}, + {etag, etag, &after, &before, errMod}, + {etag, etag, &after, &after, errMod}, + {etag, etag, &after, nil, errMod}, + {etag, etag, nil, &before, errMod}, + {etag, etag, nil, &after, errMod}, + {etag, etag, nil, nil, errMod}, + + {etag, nil, &before, &before, nil}, + {etag, nil, &before, &after, nil}, + {etag, nil, &before, nil, nil}, + {etag, nil, &after, &before, errMod}, + {etag, nil, &after, &after, errMod}, + {etag, nil, &after, nil, errMod}, + {etag, nil, nil, &before, nil}, + {etag, nil, nil, &after, nil}, + {etag, nil, nil, nil, nil}, + + {nil, getPtr("invalid_etag"), &before, &before, errCond}, + {nil, getPtr("invalid_etag"), &before, &after, nil}, + {nil, getPtr("invalid_etag"), &before, nil, nil}, + {nil, getPtr("invalid_etag"), &after, &before, errCond}, + {nil, getPtr("invalid_etag"), &after, &after, nil}, + {nil, getPtr("invalid_etag"), &after, nil, nil}, + {nil, getPtr("invalid_etag"), nil, &before, errCond}, + {nil, getPtr("invalid_etag"), nil, &after, nil}, + {nil, getPtr("invalid_etag"), nil, nil, nil}, + + {nil, etag, &before, &before, errCond}, + {nil, etag, &before, &after, errMod}, + {nil, etag, &before, nil, errMod}, + {nil, etag, &after, &before, errCond}, + {nil, etag, &after, &after, errMod}, + {nil, etag, &after, nil, errMod}, + {nil, etag, nil, &before, errCond}, + {nil, etag, nil, &after, errMod}, + {nil, etag, nil, nil, errMod}, + + {nil, nil, &before, &before, errCond}, + {nil, nil, &before, &after, nil}, + {nil, nil, &before, nil, nil}, + {nil, nil, &after, &before, errCond}, + {nil, nil, &after, &after, errMod}, + {nil, nil, &after, nil, errMod}, + {nil, nil, nil, &before, errCond}, + {nil, nil, nil, &after, nil}, + {nil, nil, nil, nil, nil}, + } { + mpKey := "mp-key" + mp, err := createMp(s3client, bucket, mpKey) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + Key: &mpKey, + UploadId: mp.UploadId, + PartNumber: getPtr(int32(1)), + CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)), + CopySourceIfMatch: test.ifmatch, + CopySourceIfNoneMatch: test.ifnonematch, + CopySourceIfModifiedSince: test.ifmodifiedsince, + CopySourceIfUnmodifiedSince: test.ifunmodifiedsince, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error { testName := "UploadPartCopy_should_copy_the_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -11427,6 +11900,87 @@ func AbortMultipartUpload_success_status_code(s *S3Conf) error { }) } +func AbortMultipartUpload_if_match_initiated_time(s *S3Conf) error { + testName := "AbortMultipartUpload_if_match_initiated_time" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + var initiated *time.Time = getPtr(time.Now()) + + // createMpUpload creates a multipart uplod + // and retruns the uploadId and creation date + abortMp := func(date *time.Time) error { + mpObj := "my-obj" + mp, err := createMp(s3client, bucket, mpObj) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + var initiatedTime *time.Time + + for _, up := range res.Uploads { + if getString(up.UploadId) == getString(mp.UploadId) { + initiatedTime = up.Initiated + break + } + } + + if initiatedTime == nil { + return fmt.Errorf("unexpected err: the multipart upload is not found") + } + + *initiated = *initiatedTime + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &mpObj, + UploadId: mp.UploadId, + IfMatchInitiatedTime: date, + }) + cancel() + + return err + } + + for i, test := range []struct { + date *time.Time + err error + }{ + {nil, nil}, + // match: success case + {initiated, nil}, + // should ignore future dates + {getPtr(initiated.AddDate(1, 0, 0)), nil}, + // should fail if the initation date doesn't match + {getPtr(initiated.AddDate(-1, 0, 1)), s3err.GetAPIError(s3err.ErrPreconditionFailed)}, + } { + err := abortMp(test.date) + if test.err == nil && err != nil { + return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err) + } + if test.err != nil { + apiErr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("invalid error type: expected s3err.APIError") + } + if err := checkApiErr(err, apiErr); err != nil { + return fmt.Errorf("test case %d failed: %w", i, err) + } + } + } + + return nil + }) +} + func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error { testName := "CompletedMultipartUpload_non_existing_bucket" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -12593,6 +13147,100 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error { }) } +func CompleteMultipartUpload_conditional_writes(s *S3Conf) error { + testName := "CompleteMultipartUpload_conditional_writes" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + etag := getPtr("") + incorrectEtag := getPtr("incorrect_etag") + errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed) + + for i, test := range []struct { + obj string + ifMatch *string + ifNoneMatch *string + err error + }{ + {obj, etag, nil, nil}, + {obj, etag, etag, errPrecond}, + {obj, etag, incorrectEtag, nil}, + {obj, incorrectEtag, incorrectEtag, errPrecond}, + {obj, incorrectEtag, etag, errPrecond}, + {obj, incorrectEtag, nil, errPrecond}, + {obj, nil, incorrectEtag, nil}, + {obj, nil, etag, errPrecond}, + {obj, nil, nil, nil}, + // should ignore the precondition headers if + // an object with the given name doesn't exist + {"obj-1", incorrectEtag, etag, nil}, + {"obj-2", etag, etag, nil}, + {"obj-3", etag, incorrectEtag, nil}, + {"obj-4", incorrectEtag, nil, nil}, + {"obj-5", nil, etag, nil}, + } { + res, err := putObjectWithData(0, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + Body: bytes.NewReader([]byte("dummy")), + }, s3client) + if err != nil { + return err + } + // azure blob storage generates different ETags for + // the exact same data. + // to avoid ETag collision reassign the etag value + *etag = *res.res.ETag + + mp, err := createMp(s3client, bucket, test.obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, test.obj, *mp.UploadId) + if err != nil { + return err + } + + part := parts[0] + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &test.obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: part.ETag, + PartNumber: getPtr(int32(1)), + ChecksumCRC64NVME: part.ChecksumCRC64NVME, + }, + }, + }, + IfMatch: test.ifMatch, + IfNoneMatch: test.ifNoneMatch, + }) + cancel() + if test.err == nil && err != nil { + return fmt.Errorf("test case %v: expected no error, instead got %w", i, err) + } + if test.err != nil { + apierr, ok := test.err.(s3err.APIError) + if !ok { + return fmt.Errorf("test case %v: invalid error type: %w", i, test.err) + } + + if err := checkApiErr(err, apierr); err != nil { + return fmt.Errorf("test case %v: %w", i, err) + } + } + } + + return nil + }) +} + func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error { testName := "CompleteMultipartUpload_invalid_part_number" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/utils.go b/tests/integration/utils.go index b57311e9..7ad0677c 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -518,11 +518,15 @@ type putObjectOutput struct { } func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) { - data := make([]byte, lgth) - rand.Read(data) - csum := sha256.Sum256(data) - r := bytes.NewReader(data) - input.Body = r + var csum [32]byte + var data []byte + if input.Body == nil { + data = make([]byte, lgth) + rand.Read(data) + csum = sha256.Sum256(data) + r := bytes.NewReader(data) + input.Body = r + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := client.PutObject(ctx, input)