mirror of
https://github.com/versity/versitygw.git
synced 2026-07-28 11:02:55 +00:00
Merge pull request #1522 from versity/sis/conditional-writes
feat: implement conditional writes
This commit is contained in:
+116
-6
@@ -301,6 +301,11 @@ func (az *Azure) PutObject(ctx context.Context, po s3response.PutObjectInput) (s
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
|
||||
err = az.evaluateWritePreconditions(ctx, po.Bucket, po.Key, po.IfMatch, po.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
|
||||
metadata := parseMetadata(po.Metadata)
|
||||
|
||||
// Store the "Expires" property in the object metadata
|
||||
@@ -850,6 +855,42 @@ func (az *Azure) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input
|
||||
}
|
||||
|
||||
func (az *Azure) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
if input.IfMatch != nil || input.IfMatchLastModifiedTime != nil || input.IfMatchSize != nil {
|
||||
// evaluate the preconditions before deleting the object
|
||||
props, err := az.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: input.Bucket,
|
||||
Key: input.Key,
|
||||
})
|
||||
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
// if object doesn't exist, skip preconditions
|
||||
// if unexpected error shows up, return the error
|
||||
return nil, err
|
||||
}
|
||||
if err == nil {
|
||||
var etag string
|
||||
if props.ETag != nil {
|
||||
etag = *props.ETag
|
||||
}
|
||||
var lastMod time.Time
|
||||
if props.LastModified != nil {
|
||||
lastMod = *props.LastModified
|
||||
}
|
||||
var size int64
|
||||
if props.ContentLength != nil {
|
||||
size = *props.ContentLength
|
||||
}
|
||||
err := backend.EvaluateObjectDeletePreconditions(etag, lastMod, size,
|
||||
backend.ObjectDeletePreconditions{
|
||||
IfMatch: input.IfMatch,
|
||||
IfMatchLastModTime: input.IfMatchLastModifiedTime,
|
||||
IfMatchSize: input.IfMatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil)
|
||||
if err != nil {
|
||||
azerr, ok := err.(*azcore.ResponseError)
|
||||
@@ -899,6 +940,26 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
if !areNils(input.CopySourceIfMatch, input.CopySourceIfNoneMatch) || !areNils(input.CopySourceIfModifiedSince, input.CopySourceIfUnmodifiedSince) {
|
||||
_, err = az.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &srcBucket,
|
||||
Key: &srcObj,
|
||||
IfMatch: input.CopySourceIfMatch,
|
||||
IfNoneMatch: input.CopySourceIfNoneMatch,
|
||||
IfModifiedSince: input.CopySourceIfModifiedSince,
|
||||
IfUnmodifiedSince: input.CopySourceIfUnmodifiedSince,
|
||||
})
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Join([]string{*input.Bucket, *input.Key}, "/") == *input.CopySource {
|
||||
if input.MetadataDirective != types.MetadataDirectiveReplace {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
|
||||
@@ -977,11 +1038,6 @@ func (az *Azure) CopyObject(ctx context.Context, input s3response.CopyObjectInpu
|
||||
}, nil
|
||||
}
|
||||
|
||||
srcBucket, srcObj, _, err := backend.ParseCopySource(*input.CopySource)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
// Get the source object
|
||||
downloadResp, err := az.client.DownloadStream(ctx, srcBucket, srcObj, nil)
|
||||
if err != nil {
|
||||
@@ -1210,7 +1266,7 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3
|
||||
func (az *Azure) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
|
||||
client, err := az.getBlockBlobClient(*input.Bucket, *input.Key)
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, nil
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
|
||||
if err := az.checkIfMpExists(ctx, *input.Bucket, *input.Key, *input.UploadId); err != nil {
|
||||
@@ -1413,6 +1469,22 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa
|
||||
// Cleans up the initiated multipart upload in .sgwtmp namespace
|
||||
func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
|
||||
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
|
||||
|
||||
if input.IfMatchInitiatedTime != nil {
|
||||
client, err := az.getBlobClient(*input.Bucket, tmpPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return azureErrToS3Err(err)
|
||||
}
|
||||
|
||||
if resp.LastModified != nil && resp.LastModified.Unix() != input.IfMatchInitiatedTime.Unix() {
|
||||
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
}
|
||||
}
|
||||
_, err := az.client.DeleteBlob(ctx, *input.Bucket, tmpPath, nil)
|
||||
if err != nil {
|
||||
return parseMpError(err)
|
||||
@@ -1440,6 +1512,11 @@ func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultip
|
||||
func (az *Azure) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
|
||||
var res s3response.CompleteMultipartUploadResult
|
||||
|
||||
err := az.evaluateWritePreconditions(ctx, input.Bucket, input.Key, input.IfMatch, input.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.CompleteMultipartUploadResult{}, "", err
|
||||
}
|
||||
|
||||
tmpPath := createMetaTmpPath(*input.Key, *input.UploadId)
|
||||
blobClient, err := az.getBlobClient(*input.Bucket, tmpPath)
|
||||
if err != nil {
|
||||
@@ -2058,6 +2135,29 @@ func (az *Azure) deleteContainerMetaData(ctx context.Context, bucket, key string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (az *Azure) evaluateWritePreconditions(ctx context.Context, bucket, object, ifMatch, ifNoneMatch *string) error {
|
||||
if areNils(ifMatch, ifNoneMatch) {
|
||||
return nil
|
||||
}
|
||||
// call HeadObject to evaluate preconditions
|
||||
// if object doesn't exist, move forward with the object creation
|
||||
// otherwise return the error
|
||||
_, err := az.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: bucket,
|
||||
Key: object,
|
||||
IfMatch: ifMatch,
|
||||
IfNoneMatch: ifNoneMatch,
|
||||
})
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNotModified)) {
|
||||
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
}
|
||||
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAclFromMetadata(meta map[string]*string, key key) (*auth.ACL, error) {
|
||||
data, ok := meta[string(key)]
|
||||
if !ok {
|
||||
@@ -2105,3 +2205,13 @@ func convertAzureEtag(etag *azcore.ETag) string {
|
||||
|
||||
return *backend.TrimEtag(str) + "-1"
|
||||
}
|
||||
|
||||
func areNils[T any](args ...*T) bool {
|
||||
for _, arg := range args {
|
||||
if arg != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -532,3 +532,41 @@ func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreCond
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions
|
||||
func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error {
|
||||
if ifMatch != nil && *ifMatch != etag {
|
||||
return errPreconditionFailed
|
||||
}
|
||||
if ifNoneMatch != nil && *ifNoneMatch == etag {
|
||||
return errPreconditionFailed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ObjectDeletePreconditions struct {
|
||||
IfMatch *string
|
||||
IfMatchLastModTime *time.Time
|
||||
IfMatchSize *int64
|
||||
}
|
||||
|
||||
// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject
|
||||
func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error {
|
||||
ifMatch := preconditions.IfMatch
|
||||
if ifMatch != nil && *ifMatch != etag {
|
||||
return errPreconditionFailed
|
||||
}
|
||||
|
||||
ifMatchTime := preconditions.IfMatchLastModTime
|
||||
if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() {
|
||||
return errPreconditionFailed
|
||||
}
|
||||
|
||||
ifMatchSize := preconditions.IfMatchSize
|
||||
if ifMatchSize != nil && *ifMatchSize != size {
|
||||
return errPreconditionFailed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
+110
-10
@@ -1401,6 +1401,14 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
return res, "", err
|
||||
}
|
||||
|
||||
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
|
||||
if err == nil {
|
||||
err = backend.EvaluateMatchPreconditions(string(b), input.IfMatch, input.IfNoneMatch)
|
||||
if err != nil {
|
||||
return res, "", err
|
||||
}
|
||||
}
|
||||
|
||||
objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID))
|
||||
@@ -2009,11 +2017,17 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
|
||||
sum := sha256.Sum256([]byte(object))
|
||||
objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
_, err = os.Stat(filepath.Join(objdir, uploadID))
|
||||
f, err := os.Stat(filepath.Join(objdir, uploadID))
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
|
||||
if mpu.IfMatchInitiatedTime != nil {
|
||||
if mpu.IfMatchInitiatedTime.Unix() != f.ModTime().Unix() {
|
||||
return s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
}
|
||||
}
|
||||
|
||||
err = os.RemoveAll(filepath.Join(objdir, uploadID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove multipart upload container: %w", err)
|
||||
@@ -2600,6 +2614,32 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
|
||||
srcf, err := os.Open(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
|
||||
}
|
||||
defer srcf.Close()
|
||||
|
||||
// evaluate preconditions
|
||||
b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey)
|
||||
srcEtag := string(b)
|
||||
if err != nil {
|
||||
srcEtag = ""
|
||||
}
|
||||
|
||||
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
|
||||
IfMatch: upi.CopySourceIfMatch,
|
||||
IfNoneMatch: upi.CopySourceIfNoneMatch,
|
||||
IfModSince: upi.CopySourceIfModifiedSince,
|
||||
IfUnmodeSince: upi.CopySourceIfUnmodifiedSince,
|
||||
})
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
|
||||
f, err := p.openTmpFile(filepath.Join(*upi.Bucket, objdir),
|
||||
*upi.Bucket, partPath, length, acct, doFalloc, p.forceNoTmpFile)
|
||||
if err != nil {
|
||||
@@ -2610,15 +2650,6 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
}
|
||||
defer f.cleanup()
|
||||
|
||||
srcf, err := os.Open(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, fmt.Errorf("open object: %w", err)
|
||||
}
|
||||
defer srcf.Close()
|
||||
|
||||
rdr := io.NewSectionReader(srcf, startOffset, length)
|
||||
hash := md5.New()
|
||||
tr := io.TeeReader(rdr, hash)
|
||||
@@ -2748,6 +2779,15 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3
|
||||
|
||||
name := filepath.Join(*po.Bucket, *po.Key)
|
||||
|
||||
// evaluate preconditions
|
||||
etagBytes, err := p.meta.RetrieveAttribute(nil, *po.Bucket, *po.Key, etagkey)
|
||||
if err == nil {
|
||||
err := backend.EvaluateMatchPreconditions(string(etagBytes), po.IfMatch, po.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
}
|
||||
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
contentLength := int64(0)
|
||||
@@ -3073,6 +3113,30 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
return nil, err
|
||||
}
|
||||
|
||||
evalPreconditions := func(f os.FileInfo, bucket, object string) error {
|
||||
var err error
|
||||
if f == nil {
|
||||
f, err = os.Stat(filepath.Join(bucket, object))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
|
||||
// evaluate preconditions
|
||||
return backend.EvaluateObjectDeletePreconditions(etag, f.ModTime(), f.Size(),
|
||||
backend.ObjectDeletePreconditions{
|
||||
IfMatch: input.IfMatch,
|
||||
IfMatchLastModTime: input.IfMatchLastModifiedTime,
|
||||
IfMatchSize: input.IfMatchSize,
|
||||
})
|
||||
}
|
||||
|
||||
// Directory objects can't have versions
|
||||
if !isDir && p.versioningEnabled() && vStatus != "" {
|
||||
if getString(input.VersionId) == "" {
|
||||
@@ -3089,6 +3153,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
err = evalPreconditions(fi, bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
acct, ok := ctx.Value("account").(auth.Account)
|
||||
if !ok {
|
||||
acct = auth.Account{}
|
||||
@@ -3148,6 +3217,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
}
|
||||
|
||||
if string(vId) == *input.VersionId {
|
||||
// evaluate preconditions
|
||||
err := evalPreconditions(nil, bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// if the specified VersionId is the same as in the latest version,
|
||||
// remove the latest version, find the latest version from the versioning
|
||||
// directory and move to the place of the deleted object, to make it the latest
|
||||
@@ -3242,6 +3316,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
}, nil
|
||||
}
|
||||
|
||||
err = evalPreconditions(nil, versionPath, *input.VersionId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
|
||||
|
||||
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
|
||||
@@ -3289,6 +3368,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
err = evalPreconditions(fi, bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = os.Remove(objpath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
@@ -4031,6 +4115,22 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey)
|
||||
srcEtag := string(b)
|
||||
if err != nil {
|
||||
srcEtag = ""
|
||||
}
|
||||
|
||||
err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
|
||||
IfMatch: input.CopySourceIfMatch,
|
||||
IfNoneMatch: input.CopySourceIfNoneMatch,
|
||||
IfModSince: input.CopySourceIfModifiedSince,
|
||||
IfUnmodeSince: input.CopySourceIfUnmodifiedSince,
|
||||
})
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
mdmap := make(map[string]string)
|
||||
p.loadObjectMetaData(nil, srcBucket, srcObject, &fi, mdmap)
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
|
||||
bucket := ctx.Params("bucket")
|
||||
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
|
||||
uploadId := ctx.Query("uploadId")
|
||||
ifMatchInitiatedTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Initiated-Time"))
|
||||
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
|
||||
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
|
||||
isBucketPublic := utils.ContextKeyPublicBucket.IsSet(ctx)
|
||||
@@ -96,9 +97,10 @@ func (c S3ApiController) AbortMultipartUpload(ctx *fiber.Ctx) (*Response, error)
|
||||
|
||||
err = c.be.AbortMultipartUpload(ctx.Context(),
|
||||
&s3.AbortMultipartUploadInput{
|
||||
UploadId: &uploadId,
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
UploadId: &uploadId,
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
IfMatchInitiatedTime: ifMatchInitiatedTime,
|
||||
})
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
@@ -113,6 +115,9 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
|
||||
versionId := ctx.Query("versionId")
|
||||
bypass := strings.EqualFold(ctx.Get("X-Amz-Bypass-Governance-Retention"), "true")
|
||||
ifMatch := utils.GetStringPtr(ctx.Get("If-Match"))
|
||||
ifMatchLastModTime := utils.ParsePreconditionDateHeader(ctx.Get("X-Amz-If-Match-Last-Modified-Time"))
|
||||
ifMatchSize := utils.ParseIfMatchSize(ctx)
|
||||
// context locals
|
||||
acct := utils.ContextKeyAccount.Get(ctx).(auth.Account)
|
||||
isRoot := utils.ContextKeyIsRoot.Get(ctx).(bool)
|
||||
@@ -165,9 +170,12 @@ func (c S3ApiController) DeleteObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
|
||||
res, err := c.be.DeleteObject(ctx.Context(),
|
||||
&s3.DeleteObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
VersionId: &versionId,
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
VersionId: &versionId,
|
||||
IfMatch: ifMatch,
|
||||
IfMatchLastModifiedTime: ifMatchLastModTime,
|
||||
IfMatchSize: ifMatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return &Response{
|
||||
|
||||
@@ -323,6 +323,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
|
||||
}, err
|
||||
}
|
||||
|
||||
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
|
||||
|
||||
res, versid, err := c.be.CompleteMultipartUpload(ctx.Context(),
|
||||
&s3.CompleteMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
@@ -338,6 +340,8 @@ func (c S3ApiController) CompleteMultipartUpload(ctx *fiber.Ctx) (*Response, err
|
||||
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
|
||||
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
|
||||
ChecksumType: checksumType,
|
||||
IfMatch: ifMatch,
|
||||
IfNoneMatch: ifNoneMatch,
|
||||
})
|
||||
return &Response{
|
||||
Data: res,
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
@@ -355,14 +354,20 @@ func (c S3ApiController) UploadPartCopy(ctx *fiber.Ctx) (*Response, error) {
|
||||
}, s3err.GetAPIError(s3err.ErrInvalidPartNumber)
|
||||
}
|
||||
|
||||
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
|
||||
|
||||
resp, err := c.be.UploadPartCopy(ctx.Context(),
|
||||
&s3.UploadPartCopyInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
CopySource: ©Source,
|
||||
PartNumber: &partNumber,
|
||||
UploadId: &uploadId,
|
||||
CopySourceRange: ©SrcRange,
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
CopySource: ©Source,
|
||||
PartNumber: &partNumber,
|
||||
UploadId: &uploadId,
|
||||
CopySourceRange: ©SrcRange,
|
||||
CopySourceIfMatch: preconditionHdrs.IfMatch,
|
||||
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
|
||||
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
|
||||
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
|
||||
})
|
||||
var headers map[string]*string
|
||||
if err == nil && resp.CopySourceVersionId != "" {
|
||||
@@ -434,10 +439,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
bucket := ctx.Params("bucket")
|
||||
key := strings.TrimPrefix(ctx.Path(), fmt.Sprintf("/%s/", bucket))
|
||||
copySource := strings.TrimPrefix(ctx.Get("X-Amz-Copy-Source"), "/")
|
||||
copySrcIfMatch := ctx.Get("X-Amz-Copy-Source-If-Match")
|
||||
copySrcIfNoneMatch := ctx.Get("X-Amz-Copy-Source-If-None-Match")
|
||||
copySrcModifSince := ctx.Get("X-Amz-Copy-Source-If-Modified-Since")
|
||||
copySrcUnmodifSince := ctx.Get("X-Amz-Copy-Source-If-Unmodified-Since")
|
||||
metaDirective := types.MetadataDirective(ctx.Get("X-Amz-Metadata-Directive", string(types.MetadataDirectiveCopy)))
|
||||
taggingDirective := types.TaggingDirective(ctx.Get("X-Amz-Tagging-Directive", string(types.TaggingDirectiveCopy)))
|
||||
contentType := ctx.Get("Content-Type")
|
||||
@@ -480,35 +481,6 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
}, err
|
||||
}
|
||||
|
||||
var mtime *time.Time
|
||||
if copySrcModifSince != "" {
|
||||
tm, err := time.Parse(iso8601Format, copySrcModifSince)
|
||||
if err != nil {
|
||||
debuglogger.Logf("error parsing copy source modified since %q: %v", copySrcModifSince, err)
|
||||
// TODO: check the error type for invalid values
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
|
||||
}
|
||||
mtime = &tm
|
||||
}
|
||||
var umtime *time.Time
|
||||
if copySrcUnmodifSince != "" {
|
||||
tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
|
||||
if err != nil {
|
||||
debuglogger.Logf("error parsing copy source unmodified since %q: %v", copySrcUnmodifSince, err)
|
||||
// TODO: check the error type for invalid values
|
||||
return &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: parsedAcl.Owner,
|
||||
},
|
||||
}, s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
|
||||
}
|
||||
umtime = &tm
|
||||
}
|
||||
|
||||
metadata := utils.GetUserMetaData(&ctx.Request().Header)
|
||||
|
||||
if metaDirective != "" && metaDirective != types.MetadataDirectiveCopy && metaDirective != types.MetadataDirectiveReplace {
|
||||
@@ -548,6 +520,8 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
}, err
|
||||
}
|
||||
|
||||
preconditionHdrs := utils.ParsePreconditionHeaders(ctx, utils.WithCopySource())
|
||||
|
||||
res, err := c.be.CopyObject(ctx.Context(),
|
||||
s3response.CopyObjectInput{
|
||||
Bucket: &bucket,
|
||||
@@ -561,10 +535,10 @@ func (c S3ApiController) CopyObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
Tagging: &tagging,
|
||||
TaggingDirective: taggingDirective,
|
||||
CopySource: ©Source,
|
||||
CopySourceIfMatch: ©SrcIfMatch,
|
||||
CopySourceIfNoneMatch: ©SrcIfNoneMatch,
|
||||
CopySourceIfModifiedSince: mtime,
|
||||
CopySourceIfUnmodifiedSince: umtime,
|
||||
CopySourceIfMatch: preconditionHdrs.IfMatch,
|
||||
CopySourceIfNoneMatch: preconditionHdrs.IfNoneMatch,
|
||||
CopySourceIfModifiedSince: preconditionHdrs.IfModSince,
|
||||
CopySourceIfUnmodifiedSince: preconditionHdrs.IfUnmodeSince,
|
||||
ExpectedBucketOwner: &acct.Access,
|
||||
Metadata: metadata,
|
||||
MetadataDirective: metaDirective,
|
||||
@@ -691,6 +665,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
body = bytes.NewReader([]byte{})
|
||||
}
|
||||
|
||||
ifMatch, ifNoneMatch := utils.ParsePreconditionMatchHeaders(ctx)
|
||||
|
||||
res, err := c.be.PutObject(ctx.Context(),
|
||||
s3response.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
@@ -714,6 +690,8 @@ func (c S3ApiController) PutObject(ctx *fiber.Ctx) (*Response, error) {
|
||||
ChecksumSHA1: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha1]),
|
||||
ChecksumSHA256: utils.GetStringPtr(checksums[types.ChecksumAlgorithmSha256]),
|
||||
ChecksumCRC64NVME: utils.GetStringPtr(checksums[types.ChecksumAlgorithmCrc64nvme]),
|
||||
IfMatch: ifMatch,
|
||||
IfNoneMatch: ifNoneMatch,
|
||||
})
|
||||
return &Response{
|
||||
Headers: map[string]*string{
|
||||
|
||||
@@ -836,42 +836,6 @@ func TestS3ApiController_CopyObject(t *testing.T) {
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid copy source modified since",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
headers: map[string]string{
|
||||
"X-Amz-Copy-Source": "bucket/object",
|
||||
"X-Amz-Copy-Source-If-Modified-Since": "invalid_date",
|
||||
},
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid copy source unmodified since",
|
||||
input: testInput{
|
||||
locals: defaultLocals,
|
||||
headers: map[string]string{
|
||||
"X-Amz-Copy-Source": "bucket/object",
|
||||
"X-Amz-Copy-Source-If-Unmodified-Since": "invalid_date",
|
||||
},
|
||||
},
|
||||
output: testOutput{
|
||||
response: &Response{
|
||||
MetaOpts: &MetaOptions{
|
||||
BucketOwner: "root",
|
||||
},
|
||||
},
|
||||
err: s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid metadata directive",
|
||||
input: testInput{
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/debuglogger"
|
||||
)
|
||||
|
||||
// ConditionalHeaders holds the conditional header values
|
||||
type ConditionalHeaders struct {
|
||||
IfMatch *string
|
||||
IfNoneMatch *string
|
||||
IfModSince *time.Time
|
||||
IfUnmodeSince *time.Time
|
||||
}
|
||||
|
||||
type precondtionCfg struct {
|
||||
withCopySource bool
|
||||
}
|
||||
|
||||
type preconditionOpt func(*precondtionCfg)
|
||||
|
||||
func WithCopySource() preconditionOpt {
|
||||
return func(o *precondtionCfg) { o.withCopySource = true }
|
||||
}
|
||||
|
||||
// ParsePreconditionHeaders parses the precondition headers:
|
||||
// - If-Match
|
||||
// - If-None-Match
|
||||
// - If-Modified-Since
|
||||
// - If-Unmodified-Since
|
||||
func ParsePreconditionHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) ConditionalHeaders {
|
||||
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx, opts...)
|
||||
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx, opts...)
|
||||
|
||||
return ConditionalHeaders{
|
||||
IfMatch: ifMatch,
|
||||
IfNoneMatch: ifNoneMatch,
|
||||
IfModSince: ifModSince,
|
||||
IfUnmodeSince: ifUnmodeSince,
|
||||
}
|
||||
}
|
||||
|
||||
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
|
||||
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*string, *string) {
|
||||
cfg := new(precondtionCfg)
|
||||
for _, opt := range opts {
|
||||
opt(cfg)
|
||||
}
|
||||
prefix := ""
|
||||
if cfg.withCopySource {
|
||||
prefix = "X-Amz-Copy-Source-"
|
||||
}
|
||||
return GetStringPtr(ctx.Get(prefix + "If-Match")), GetStringPtr(ctx.Get(prefix + "If-None-Match"))
|
||||
}
|
||||
|
||||
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
|
||||
// headers from fiber context to *time.Time
|
||||
func ParsePreconditionDateHeaders(ctx *fiber.Ctx, opts ...preconditionOpt) (*time.Time, *time.Time) {
|
||||
cfg := new(precondtionCfg)
|
||||
for _, opt := range opts {
|
||||
opt(cfg)
|
||||
}
|
||||
prefix := ""
|
||||
if cfg.withCopySource {
|
||||
prefix = "X-Amz-Copy-Source-"
|
||||
}
|
||||
|
||||
ifModSince := ctx.Get(prefix + "If-Modified-Since")
|
||||
ifUnmodSince := ctx.Get(prefix + "If-Unmodified-Since")
|
||||
|
||||
ifModSinceParsed := ParsePreconditionDateHeader(ifModSince)
|
||||
ifUnmodSinceParsed := ParsePreconditionDateHeader(ifUnmodSince)
|
||||
|
||||
return ifModSinceParsed, ifUnmodSinceParsed
|
||||
}
|
||||
|
||||
// ParsePreconditionDateHeader tries to parse the given date string as
|
||||
// - RFC1123
|
||||
// - RFC3339
|
||||
// both are valid
|
||||
func ParsePreconditionDateHeader(date string) *time.Time {
|
||||
if date == "" {
|
||||
return nil
|
||||
}
|
||||
// try to parse as RFC1123
|
||||
parsed, err := time.Parse(time.RFC1123, date)
|
||||
if err == nil {
|
||||
// ignore future dates
|
||||
if parsed.After(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &parsed
|
||||
}
|
||||
|
||||
// try to parse as RFC3339
|
||||
parsed, err = time.Parse(time.RFC3339, date)
|
||||
if err == nil {
|
||||
// ignore future dates
|
||||
if parsed.After(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &parsed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseIfMatchSize parses the 'x-amz-if-match-size' to *int64
|
||||
// if parsing fails, returns nil
|
||||
func ParseIfMatchSize(ctx *fiber.Ctx) *int64 {
|
||||
ifMatchSizeHdr := ctx.Get("x-amz-if-match-size")
|
||||
ifMatchSize, err := strconv.ParseInt(ifMatchSizeHdr, 10, 64)
|
||||
if err != nil {
|
||||
debuglogger.Logf("failed to parse 'x-amz-if-match-size': %s", ifMatchSizeHdr)
|
||||
return nil
|
||||
}
|
||||
|
||||
return &ifMatchSize
|
||||
}
|
||||
@@ -642,59 +642,6 @@ func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, type
|
||||
return algo, chType, nil
|
||||
}
|
||||
|
||||
// ConditionalHeaders holds the conditional header values
|
||||
type ConditionalHeaders struct {
|
||||
IfMatch *string
|
||||
IfNoneMatch *string
|
||||
IfModSince *time.Time
|
||||
IfUnmodeSince *time.Time
|
||||
}
|
||||
|
||||
// ParsePreconditionHeaders parses the precondition headers:
|
||||
// - If-Match
|
||||
// - If-None-Match
|
||||
// - If-Modified-Since
|
||||
// - If-Unmodified-Since
|
||||
func ParsePreconditionHeaders(ctx *fiber.Ctx) ConditionalHeaders {
|
||||
ifMatch, ifNoneMatch := ParsePreconditionMatchHeaders(ctx)
|
||||
ifModSince, ifUnmodeSince := ParsePreconditionDateHeaders(ctx)
|
||||
|
||||
return ConditionalHeaders{
|
||||
IfMatch: ifMatch,
|
||||
IfNoneMatch: ifNoneMatch,
|
||||
IfModSince: ifModSince,
|
||||
IfUnmodeSince: ifUnmodeSince,
|
||||
}
|
||||
}
|
||||
|
||||
// ParsePreconditionMatchHeaders extracts "If-Match" and "If-None-Match" headers from fiber Ctx
|
||||
func ParsePreconditionMatchHeaders(ctx *fiber.Ctx) (*string, *string) {
|
||||
return GetStringPtr(ctx.Get("If-Match")), GetStringPtr(ctx.Get("If-None-Match"))
|
||||
}
|
||||
|
||||
// ParsePreconditionDateHeaders parses the "If-Modified-Since" and "If-Unmodified-Since"
|
||||
// headers from fiber context to *time.Time
|
||||
func ParsePreconditionDateHeaders(ctx *fiber.Ctx) (*time.Time, *time.Time) {
|
||||
ifModSince := ctx.Get("If-Modified-Since")
|
||||
ifUnmodSince := ctx.Get("If-Unmodified-Since")
|
||||
|
||||
var ifModSinceParsed, ifUnmodSinceParsed *time.Time
|
||||
|
||||
// the time format should be a valid RFC1123
|
||||
// if parsing fails, ignore the error and leave the value as nil
|
||||
modParsed, err := time.Parse(time.RFC1123, ifModSince)
|
||||
if err == nil {
|
||||
ifModSinceParsed = &modParsed
|
||||
}
|
||||
|
||||
unmodParsed, err := time.Parse(time.RFC1123, ifUnmodSince)
|
||||
if err == nil {
|
||||
ifUnmodSinceParsed = &unmodParsed
|
||||
}
|
||||
|
||||
return ifModSinceParsed, ifUnmodSinceParsed
|
||||
}
|
||||
|
||||
// TagLimit specifies the allowed tag count in a tag set
|
||||
type TagLimit int
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ func TestPutObject(s *S3Conf) {
|
||||
PutObject_with_object_lock(s)
|
||||
PutObject_invalid_legal_hold(s)
|
||||
PutObject_invalid_object_lock_mode(s)
|
||||
PutObject_conditional_writes(s)
|
||||
//TODO: remove the condition after implementing checksums in azure
|
||||
if !s.azureTests {
|
||||
PutObject_checksum_algorithm_and_header_mismatch(s)
|
||||
@@ -279,6 +280,7 @@ func TestDeleteObject(s *S3Conf) {
|
||||
DeleteObject_non_existing_dir_object(s)
|
||||
DeleteObject_directory_object(s)
|
||||
DeleteObject_non_empty_dir_obj(s)
|
||||
DeleteObject_conditional_writes(s)
|
||||
DeleteObject_success(s)
|
||||
DeleteObject_success_status_code(s)
|
||||
}
|
||||
@@ -307,6 +309,7 @@ func TestCopyObject(s *S3Conf) {
|
||||
CopyObject_invalid_object_lock_mode(s)
|
||||
CopyObject_with_legal_hold(s)
|
||||
CopyObject_with_retention_lock(s)
|
||||
CopyObject_conditional_reads(s)
|
||||
//TODO: remove the condition after implementing checksums in azure
|
||||
if !s.azureTests {
|
||||
CopyObject_invalid_checksum_algorithm(s)
|
||||
@@ -398,6 +401,7 @@ func TestUploadPartCopy(s *S3Conf) {
|
||||
UploadPartCopy_should_copy_the_checksum(s)
|
||||
UploadPartCopy_should_not_copy_the_checksum(s)
|
||||
UploadPartCopy_should_calculate_the_checksum(s)
|
||||
UploadPartCopy_conditional_reads(s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,6 +441,7 @@ func TestAbortMultipartUpload(s *S3Conf) {
|
||||
AbortMultipartUpload_incorrect_object_key(s)
|
||||
AbortMultipartUpload_success(s)
|
||||
AbortMultipartUpload_success_status_code(s)
|
||||
AbortMultipartUpload_if_match_initiated_time(s)
|
||||
}
|
||||
|
||||
func TestCompleteMultipartUpload(s *S3Conf) {
|
||||
@@ -448,6 +453,7 @@ func TestCompleteMultipartUpload(s *S3Conf) {
|
||||
CompleteMultipartUpload_empty_parts(s)
|
||||
CompleteMultipartUpload_incorrect_parts_order(s)
|
||||
CompleteMultipartUpload_mpu_object_size(s)
|
||||
CompleteMultipartUpload_conditional_writes(s)
|
||||
//TODO: remove the condition after implementing checksums in azure
|
||||
if !s.azureTests {
|
||||
CompleteMultipartUpload_invalid_checksum_type(s)
|
||||
@@ -1053,6 +1059,7 @@ func GetIntTests() IntTests {
|
||||
"PutObject_with_object_lock": PutObject_with_object_lock,
|
||||
"PutObject_invalid_legal_hold": PutObject_invalid_legal_hold,
|
||||
"PutObject_invalid_object_lock_mode": PutObject_invalid_object_lock_mode,
|
||||
"PutObject_conditional_writes": PutObject_conditional_writes,
|
||||
"PutObject_invalid_credentials": PutObject_invalid_credentials,
|
||||
"PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch,
|
||||
"PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers,
|
||||
@@ -1184,6 +1191,7 @@ func GetIntTests() IntTests {
|
||||
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
|
||||
"DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash,
|
||||
"DeleteObject_non_empty_dir_obj": DeleteObject_non_empty_dir_obj,
|
||||
"DeleteObject_conditional_writes": DeleteObject_conditional_writes,
|
||||
"DeleteObject_name_too_long": DeleteObject_name_too_long,
|
||||
"CopyObject_overwrite_same_dir_object": CopyObject_overwrite_same_dir_object,
|
||||
"CopyObject_overwrite_same_file_object": CopyObject_overwrite_same_file_object,
|
||||
@@ -1211,6 +1219,7 @@ func GetIntTests() IntTests {
|
||||
"CopyObject_invalid_object_lock_mode": CopyObject_invalid_object_lock_mode,
|
||||
"CopyObject_with_legal_hold": CopyObject_with_legal_hold,
|
||||
"CopyObject_with_retention_lock": CopyObject_with_retention_lock,
|
||||
"CopyObject_conditional_reads": CopyObject_conditional_reads,
|
||||
"CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm,
|
||||
"CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy,
|
||||
"CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum,
|
||||
@@ -1270,6 +1279,7 @@ func GetIntTests() IntTests {
|
||||
"UploadPartCopy_exceeding_copy_source_range": UploadPartCopy_exceeding_copy_source_range,
|
||||
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
|
||||
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
|
||||
"UploadPartCopy_conditional_reads": UploadPartCopy_conditional_reads,
|
||||
"UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum,
|
||||
"UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum,
|
||||
"UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum,
|
||||
@@ -1295,6 +1305,7 @@ func GetIntTests() IntTests {
|
||||
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
|
||||
"AbortMultipartUpload_success": AbortMultipartUpload_success,
|
||||
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
|
||||
"AbortMultipartUpload_if_match_initiated_time": AbortMultipartUpload_if_match_initiated_time,
|
||||
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
|
||||
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
|
||||
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
|
||||
@@ -1302,6 +1313,7 @@ func GetIntTests() IntTests {
|
||||
"CompleteMultipartUpload_empty_parts": CompleteMultipartUpload_empty_parts,
|
||||
"CompleteMultipartUpload_incorrect_parts_order": CompleteMultipartUpload_incorrect_parts_order,
|
||||
"CompleteMultipartUpload_mpu_object_size": CompleteMultipartUpload_mpu_object_size,
|
||||
"CompleteMultipartUpload_conditional_writes": CompleteMultipartUpload_conditional_writes,
|
||||
"CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type,
|
||||
"CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part,
|
||||
"CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part,
|
||||
|
||||
@@ -3043,6 +3043,7 @@ func PutObject_invalid_legal_hold(s *S3Conf) error {
|
||||
return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus))
|
||||
}, withLock())
|
||||
}
|
||||
|
||||
func PutObject_invalid_object_lock_mode(s *S3Conf) error {
|
||||
testName := "PutObject_invalid_object_lock_mode"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -3057,6 +3058,78 @@ func PutObject_invalid_object_lock_mode(s *S3Conf) error {
|
||||
}, withLock())
|
||||
}
|
||||
|
||||
func PutObject_conditional_writes(s *S3Conf) error {
|
||||
testName := "PutObject_conditional_writes"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj := "my-obj"
|
||||
res, err := putObjectWithData(0, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
Body: bytes.NewReader([]byte("dummy")),
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
etag := res.res.ETag
|
||||
incorrectEtag := getPtr("incorrect_etag")
|
||||
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
|
||||
for i, test := range []struct {
|
||||
obj string
|
||||
ifMatch *string
|
||||
ifNoneMatch *string
|
||||
err error
|
||||
}{
|
||||
{obj, etag, nil, nil},
|
||||
{obj, etag, etag, errPrecond},
|
||||
{obj, etag, incorrectEtag, nil},
|
||||
{obj, incorrectEtag, incorrectEtag, errPrecond},
|
||||
{obj, incorrectEtag, etag, errPrecond},
|
||||
{obj, incorrectEtag, nil, errPrecond},
|
||||
{obj, nil, incorrectEtag, nil},
|
||||
{obj, nil, etag, errPrecond},
|
||||
{obj, nil, nil, nil},
|
||||
// should ignore the precondition headers if
|
||||
// an object with the given name doesn't exist
|
||||
{"obj-1", incorrectEtag, etag, nil},
|
||||
{"obj-2", etag, etag, nil},
|
||||
{"obj-3", etag, incorrectEtag, nil},
|
||||
{"obj-4", incorrectEtag, nil, nil},
|
||||
{"obj-5", nil, etag, nil},
|
||||
} {
|
||||
res, err := putObjectWithData(0, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &test.obj,
|
||||
Body: bytes.NewReader([]byte("dummy")),
|
||||
IfMatch: test.ifMatch,
|
||||
IfNoneMatch: test.ifNoneMatch,
|
||||
}, s3client)
|
||||
if err == nil {
|
||||
// azure blob storage generates different ETags for
|
||||
// the exact same data.
|
||||
// to avoid ETag collision reassign the etag value
|
||||
*etag = *res.res.ETag
|
||||
}
|
||||
if test.err == nil && err != nil {
|
||||
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
|
||||
}
|
||||
if test.err != nil {
|
||||
apierr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
|
||||
}
|
||||
|
||||
if err := checkApiErr(err, apierr); err != nil {
|
||||
return fmt.Errorf("test case %v: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func PutObject_checksum_algorithm_and_header_mismatch(s *S3Conf) error {
|
||||
testName := "PutObject_checksum_algorithm_and_header_mismatch"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -6895,6 +6968,98 @@ func DeleteObject_non_empty_dir_obj(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObject_conditional_writes(s *S3Conf) error {
|
||||
testName := "DeleteObject_conditional_writes"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj := "my-obj"
|
||||
var etag *string = getPtr("")
|
||||
var size *int64 = getPtr(int64(0))
|
||||
var modTime *time.Time = getPtr(time.Now())
|
||||
|
||||
createObj := func() error {
|
||||
res, err := putObjectWithData(0, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
Body: bytes.NewReader([]byte("dummy")),
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get the exact LastModified time
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*etag = *res.res.ETag
|
||||
*size = *res.res.Size
|
||||
*modTime = *out.LastModified
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err := createObj()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
|
||||
for i, test := range []struct {
|
||||
ifMatch *string
|
||||
size *int64
|
||||
modTime *time.Time
|
||||
err error
|
||||
}{
|
||||
// no error cases
|
||||
{etag, size, modTime, nil},
|
||||
{etag, nil, nil, nil},
|
||||
{nil, size, nil, nil},
|
||||
{nil, nil, modTime, nil},
|
||||
{etag, size, nil, nil},
|
||||
{etag, nil, modTime, nil},
|
||||
{nil, size, modTime, nil},
|
||||
// error cases
|
||||
{getPtr("incorrect_etag"), nil, nil, errPrecond},
|
||||
{nil, getPtr(int64(23234)), nil, errPrecond},
|
||||
{nil, nil, getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
|
||||
{getPtr("incorrect_etag"), getPtr(int64(23234)), nil, errPrecond},
|
||||
{getPtr("incorrect_etag"), getPtr(int64(23234)), getPtr(time.Now().AddDate(-1, -1, -1)), errPrecond},
|
||||
} {
|
||||
err := createObj()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
IfMatch: test.ifMatch,
|
||||
IfMatchSize: test.size,
|
||||
IfMatchLastModifiedTime: test.modTime,
|
||||
})
|
||||
cancel()
|
||||
if test.err != nil {
|
||||
apiErr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid error type: expected s3err.APIError")
|
||||
}
|
||||
if err := checkApiErr(err, apiErr); err != nil {
|
||||
return fmt.Errorf("test case %d failed: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteObject_directory_not_empty(s *S3Conf) error {
|
||||
testName := "DeleteObject_directory_not_empty"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -8022,6 +8187,156 @@ func CopyObject_with_retention_lock(s *S3Conf) error {
|
||||
}, withLock())
|
||||
}
|
||||
|
||||
func CopyObject_conditional_reads(s *S3Conf) error {
|
||||
testName := "CopyObject_conditional_reads"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
key := "my-obj"
|
||||
obj, err := putObjectWithData(10, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errMod := s3err.GetAPIError(s3err.ErrNotModified)
|
||||
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
|
||||
// sleep one second to get dates before and after
|
||||
// the object creation
|
||||
time.Sleep(time.Second * 1)
|
||||
|
||||
before := time.Now().AddDate(0, 0, -3)
|
||||
after := time.Now()
|
||||
etag := obj.res.ETag
|
||||
|
||||
for i, test := range []struct {
|
||||
ifmatch *string
|
||||
ifnonematch *string
|
||||
ifmodifiedsince *time.Time
|
||||
ifunmodifiedsince *time.Time
|
||||
err error
|
||||
}{
|
||||
// all the cases when preconditions are either empty, true or false
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
|
||||
|
||||
{getPtr("invalid_etag"), etag, &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, nil, errCond},
|
||||
|
||||
{getPtr("invalid_etag"), nil, &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, nil, errCond},
|
||||
|
||||
{etag, getPtr("invalid_etag"), &before, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), &before, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), &before, nil, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, nil, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, nil, nil},
|
||||
|
||||
{etag, etag, &before, &before, errMod},
|
||||
{etag, etag, &before, &after, errMod},
|
||||
{etag, etag, &before, nil, errMod},
|
||||
{etag, etag, &after, &before, errMod},
|
||||
{etag, etag, &after, &after, errMod},
|
||||
{etag, etag, &after, nil, errMod},
|
||||
{etag, etag, nil, &before, errMod},
|
||||
{etag, etag, nil, &after, errMod},
|
||||
{etag, etag, nil, nil, errMod},
|
||||
|
||||
{etag, nil, &before, &before, nil},
|
||||
{etag, nil, &before, &after, nil},
|
||||
{etag, nil, &before, nil, nil},
|
||||
{etag, nil, &after, &before, errMod},
|
||||
{etag, nil, &after, &after, errMod},
|
||||
{etag, nil, &after, nil, errMod},
|
||||
{etag, nil, nil, &before, nil},
|
||||
{etag, nil, nil, &after, nil},
|
||||
{etag, nil, nil, nil, nil},
|
||||
|
||||
{nil, getPtr("invalid_etag"), &before, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), &before, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), &before, nil, nil},
|
||||
{nil, getPtr("invalid_etag"), &after, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), &after, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), &after, nil, nil},
|
||||
{nil, getPtr("invalid_etag"), nil, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), nil, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), nil, nil, nil},
|
||||
|
||||
{nil, etag, &before, &before, errCond},
|
||||
{nil, etag, &before, &after, errMod},
|
||||
{nil, etag, &before, nil, errMod},
|
||||
{nil, etag, &after, &before, errCond},
|
||||
{nil, etag, &after, &after, errMod},
|
||||
{nil, etag, &after, nil, errMod},
|
||||
{nil, etag, nil, &before, errCond},
|
||||
{nil, etag, nil, &after, errMod},
|
||||
{nil, etag, nil, nil, errMod},
|
||||
|
||||
{nil, nil, &before, &before, errCond},
|
||||
{nil, nil, &before, &after, nil},
|
||||
{nil, nil, &before, nil, nil},
|
||||
{nil, nil, &after, &before, errCond},
|
||||
{nil, nil, &after, &after, errMod},
|
||||
{nil, nil, &after, nil, errMod},
|
||||
{nil, nil, nil, &before, errCond},
|
||||
{nil, nil, nil, &after, nil},
|
||||
{nil, nil, nil, nil, nil},
|
||||
} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr("dst-obj"),
|
||||
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
|
||||
CopySourceIfMatch: test.ifmatch,
|
||||
CopySourceIfNoneMatch: test.ifnonematch,
|
||||
CopySourceIfModifiedSince: test.ifmodifiedsince,
|
||||
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
|
||||
})
|
||||
cancel()
|
||||
if test.err == nil && err != nil {
|
||||
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
|
||||
}
|
||||
if test.err != nil {
|
||||
apiErr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid error type: expected s3err.APIError")
|
||||
}
|
||||
if err := checkApiErr(err, apiErr); err != nil {
|
||||
return fmt.Errorf("test case %d failed: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CopyObject_invalid_checksum_algorithm(s *S3Conf) error {
|
||||
testName := "CopyObject_invalid_checksum_algorithm"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -10476,6 +10791,164 @@ func UploadPartCopy_by_range_success(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func UploadPartCopy_conditional_reads(s *S3Conf) error {
|
||||
testName := "UploadPartCopy_conditional_reads"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
key := "my-obj"
|
||||
obj, err := putObjectWithData(10, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errMod := s3err.GetAPIError(s3err.ErrNotModified)
|
||||
errCond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
|
||||
// sleep one second to get dates before and after
|
||||
// the object creation
|
||||
time.Sleep(time.Second * 1)
|
||||
|
||||
before := time.Now().AddDate(0, 0, -3)
|
||||
after := time.Now()
|
||||
etag := obj.res.ETag
|
||||
|
||||
for i, test := range []struct {
|
||||
ifmatch *string
|
||||
ifnonematch *string
|
||||
ifmodifiedsince *time.Time
|
||||
ifunmodifiedsince *time.Time
|
||||
err error
|
||||
}{
|
||||
// all the cases when preconditions are either empty, true or false
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), getPtr("invalid_etag"), nil, nil, errCond},
|
||||
|
||||
{getPtr("invalid_etag"), etag, &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), etag, nil, nil, errCond},
|
||||
|
||||
{getPtr("invalid_etag"), nil, &before, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, &before, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, &before, nil, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, &after, nil, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, &before, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, &after, errCond},
|
||||
{getPtr("invalid_etag"), nil, nil, nil, errCond},
|
||||
|
||||
{etag, getPtr("invalid_etag"), &before, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), &before, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), &before, nil, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), &after, nil, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, &before, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, &after, nil},
|
||||
{etag, getPtr("invalid_etag"), nil, nil, nil},
|
||||
|
||||
{etag, etag, &before, &before, errMod},
|
||||
{etag, etag, &before, &after, errMod},
|
||||
{etag, etag, &before, nil, errMod},
|
||||
{etag, etag, &after, &before, errMod},
|
||||
{etag, etag, &after, &after, errMod},
|
||||
{etag, etag, &after, nil, errMod},
|
||||
{etag, etag, nil, &before, errMod},
|
||||
{etag, etag, nil, &after, errMod},
|
||||
{etag, etag, nil, nil, errMod},
|
||||
|
||||
{etag, nil, &before, &before, nil},
|
||||
{etag, nil, &before, &after, nil},
|
||||
{etag, nil, &before, nil, nil},
|
||||
{etag, nil, &after, &before, errMod},
|
||||
{etag, nil, &after, &after, errMod},
|
||||
{etag, nil, &after, nil, errMod},
|
||||
{etag, nil, nil, &before, nil},
|
||||
{etag, nil, nil, &after, nil},
|
||||
{etag, nil, nil, nil, nil},
|
||||
|
||||
{nil, getPtr("invalid_etag"), &before, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), &before, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), &before, nil, nil},
|
||||
{nil, getPtr("invalid_etag"), &after, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), &after, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), &after, nil, nil},
|
||||
{nil, getPtr("invalid_etag"), nil, &before, errCond},
|
||||
{nil, getPtr("invalid_etag"), nil, &after, nil},
|
||||
{nil, getPtr("invalid_etag"), nil, nil, nil},
|
||||
|
||||
{nil, etag, &before, &before, errCond},
|
||||
{nil, etag, &before, &after, errMod},
|
||||
{nil, etag, &before, nil, errMod},
|
||||
{nil, etag, &after, &before, errCond},
|
||||
{nil, etag, &after, &after, errMod},
|
||||
{nil, etag, &after, nil, errMod},
|
||||
{nil, etag, nil, &before, errCond},
|
||||
{nil, etag, nil, &after, errMod},
|
||||
{nil, etag, nil, nil, errMod},
|
||||
|
||||
{nil, nil, &before, &before, errCond},
|
||||
{nil, nil, &before, &after, nil},
|
||||
{nil, nil, &before, nil, nil},
|
||||
{nil, nil, &after, &before, errCond},
|
||||
{nil, nil, &after, &after, errMod},
|
||||
{nil, nil, &after, nil, errMod},
|
||||
{nil, nil, nil, &before, errCond},
|
||||
{nil, nil, nil, &after, nil},
|
||||
{nil, nil, nil, nil, nil},
|
||||
} {
|
||||
mpKey := "mp-key"
|
||||
mp, err := createMp(s3client, bucket, mpKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
|
||||
Bucket: &bucket,
|
||||
Key: &mpKey,
|
||||
UploadId: mp.UploadId,
|
||||
PartNumber: getPtr(int32(1)),
|
||||
CopySource: getPtr(fmt.Sprintf("%s/%s", bucket, key)),
|
||||
CopySourceIfMatch: test.ifmatch,
|
||||
CopySourceIfNoneMatch: test.ifnonematch,
|
||||
CopySourceIfModifiedSince: test.ifmodifiedsince,
|
||||
CopySourceIfUnmodifiedSince: test.ifunmodifiedsince,
|
||||
})
|
||||
cancel()
|
||||
if test.err == nil && err != nil {
|
||||
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
|
||||
}
|
||||
if test.err != nil {
|
||||
apiErr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid error type: expected s3err.APIError")
|
||||
}
|
||||
if err := checkApiErr(err, apiErr); err != nil {
|
||||
return fmt.Errorf("test case %d failed: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func UploadPartCopy_should_copy_the_checksum(s *S3Conf) error {
|
||||
testName := "UploadPartCopy_should_copy_the_checksum"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -11427,6 +11900,87 @@ func AbortMultipartUpload_success_status_code(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func AbortMultipartUpload_if_match_initiated_time(s *S3Conf) error {
|
||||
testName := "AbortMultipartUpload_if_match_initiated_time"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
var initiated *time.Time = getPtr(time.Now())
|
||||
|
||||
// createMpUpload creates a multipart uplod
|
||||
// and retruns the uploadId and creation date
|
||||
abortMp := func(date *time.Time) error {
|
||||
mpObj := "my-obj"
|
||||
mp, err := createMp(s3client, bucket, mpObj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var initiatedTime *time.Time
|
||||
|
||||
for _, up := range res.Uploads {
|
||||
if getString(up.UploadId) == getString(mp.UploadId) {
|
||||
initiatedTime = up.Initiated
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if initiatedTime == nil {
|
||||
return fmt.Errorf("unexpected err: the multipart upload is not found")
|
||||
}
|
||||
|
||||
*initiated = *initiatedTime
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: &mpObj,
|
||||
UploadId: mp.UploadId,
|
||||
IfMatchInitiatedTime: date,
|
||||
})
|
||||
cancel()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
for i, test := range []struct {
|
||||
date *time.Time
|
||||
err error
|
||||
}{
|
||||
{nil, nil},
|
||||
// match: success case
|
||||
{initiated, nil},
|
||||
// should ignore future dates
|
||||
{getPtr(initiated.AddDate(1, 0, 0)), nil},
|
||||
// should fail if the initation date doesn't match
|
||||
{getPtr(initiated.AddDate(-1, 0, 1)), s3err.GetAPIError(s3err.ErrPreconditionFailed)},
|
||||
} {
|
||||
err := abortMp(test.date)
|
||||
if test.err == nil && err != nil {
|
||||
return fmt.Errorf("test case %d failed: expected no error, but got %v", i, err)
|
||||
}
|
||||
if test.err != nil {
|
||||
apiErr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid error type: expected s3err.APIError")
|
||||
}
|
||||
if err := checkApiErr(err, apiErr); err != nil {
|
||||
return fmt.Errorf("test case %d failed: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) error {
|
||||
testName := "CompletedMultipartUpload_non_existing_bucket"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
@@ -12593,6 +13147,100 @@ func CompleteMultipartUpload_mpu_object_size(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func CompleteMultipartUpload_conditional_writes(s *S3Conf) error {
|
||||
testName := "CompleteMultipartUpload_conditional_writes"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
obj := "my-obj"
|
||||
|
||||
etag := getPtr("")
|
||||
incorrectEtag := getPtr("incorrect_etag")
|
||||
errPrecond := s3err.GetAPIError(s3err.ErrPreconditionFailed)
|
||||
|
||||
for i, test := range []struct {
|
||||
obj string
|
||||
ifMatch *string
|
||||
ifNoneMatch *string
|
||||
err error
|
||||
}{
|
||||
{obj, etag, nil, nil},
|
||||
{obj, etag, etag, errPrecond},
|
||||
{obj, etag, incorrectEtag, nil},
|
||||
{obj, incorrectEtag, incorrectEtag, errPrecond},
|
||||
{obj, incorrectEtag, etag, errPrecond},
|
||||
{obj, incorrectEtag, nil, errPrecond},
|
||||
{obj, nil, incorrectEtag, nil},
|
||||
{obj, nil, etag, errPrecond},
|
||||
{obj, nil, nil, nil},
|
||||
// should ignore the precondition headers if
|
||||
// an object with the given name doesn't exist
|
||||
{"obj-1", incorrectEtag, etag, nil},
|
||||
{"obj-2", etag, etag, nil},
|
||||
{"obj-3", etag, incorrectEtag, nil},
|
||||
{"obj-4", incorrectEtag, nil, nil},
|
||||
{"obj-5", nil, etag, nil},
|
||||
} {
|
||||
res, err := putObjectWithData(0, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &obj,
|
||||
Body: bytes.NewReader([]byte("dummy")),
|
||||
}, s3client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// azure blob storage generates different ETags for
|
||||
// the exact same data.
|
||||
// to avoid ETag collision reassign the etag value
|
||||
*etag = *res.res.ETag
|
||||
|
||||
mp, err := createMp(s3client, bucket, test.obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, test.obj, *mp.UploadId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
part := parts[0]
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: &test.obj,
|
||||
UploadId: mp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{
|
||||
ETag: part.ETag,
|
||||
PartNumber: getPtr(int32(1)),
|
||||
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
|
||||
},
|
||||
},
|
||||
},
|
||||
IfMatch: test.ifMatch,
|
||||
IfNoneMatch: test.ifNoneMatch,
|
||||
})
|
||||
cancel()
|
||||
if test.err == nil && err != nil {
|
||||
return fmt.Errorf("test case %v: expected no error, instead got %w", i, err)
|
||||
}
|
||||
if test.err != nil {
|
||||
apierr, ok := test.err.(s3err.APIError)
|
||||
if !ok {
|
||||
return fmt.Errorf("test case %v: invalid error type: %w", i, test.err)
|
||||
}
|
||||
|
||||
if err := checkApiErr(err, apierr); err != nil {
|
||||
return fmt.Errorf("test case %v: %w", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CompleteMultipartUpload_invalid_part_number(s *S3Conf) error {
|
||||
testName := "CompleteMultipartUpload_invalid_part_number"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
|
||||
@@ -518,11 +518,15 @@ type putObjectOutput struct {
|
||||
}
|
||||
|
||||
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (*putObjectOutput, error) {
|
||||
data := make([]byte, lgth)
|
||||
rand.Read(data)
|
||||
csum := sha256.Sum256(data)
|
||||
r := bytes.NewReader(data)
|
||||
input.Body = r
|
||||
var csum [32]byte
|
||||
var data []byte
|
||||
if input.Body == nil {
|
||||
data = make([]byte, lgth)
|
||||
rand.Read(data)
|
||||
csum = sha256.Sum256(data)
|
||||
r := bytes.NewReader(data)
|
||||
input.Body = r
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := client.PutObject(ctx, input)
|
||||
|
||||
Reference in New Issue
Block a user